HUGE WebUI overhaul

- Removed ugly styles in header
- Properly implemented Bootstrap's NavBar
- Added username to session

.gitignore

@@ -158,5 +158,6 @@ cert/*
 !cert/server.pem
 config/*
 deliver/*
+*.gz
 
 dbdump-*.json

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.9.15-slim-bullseye
+
+RUN apt update && apt install default-libmysqlclient-dev build-essential libtk nodejs npm -y
+
+WORKDIR /app
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+RUN npm i -g nodemon
+
+COPY entrypoint.sh entrypoint.sh
+RUN chmod +x entrypoint.sh
+
+COPY index.py index.py
+COPY dbutils.py dbutils.py
+ADD core core
+ADD titles titles
+ADD config config
+ADD log log
+ADD cert cert
+
+ENTRYPOINT [ "/app/entrypoint.sh" ]

changelog.md

@@ -0,0 +1,99 @@
+# Changelog
+Documenting updates to ARTEMiS, to be updated every time the master branch is pushed to.
+
+## 20230716
+### General
++ Docker files added (#19)
++ Added support for threading
+  + This comes with the caviat that enabling it will not allow you to use Ctrl + C to stop the server.
+
+### Webui
++ Small improvements
++ Add card display
+
+### Allnet
++ Billing format validation
++ Fix naomitest.html endpoint
++ Add event logging for auths and billing
++ LoaderStateRecorder endpoint handler added
+
+### Mucha
++ Fixed log level always being "Info"
++ Add stub handler for DownloadState
+
+### Sword Art Online
++ Support added
+
+### Crossbeats
++ Added threading to profile loading
+  + This should cause a noticeable speed-up
+
+### Card Maker
++ DX Passes fixed
++ Various improvements
+
+### Diva
++ Added clear status calculation
++ Various minor fixes and improvements
+
+### Maimai
++ Added support for memorial photo uploads
++ Added support for the following versions
+  + Festival
+  + FiNALE
++ Various bug fixes and improvements
+
+### Wacca
++ Fixed an error that sometimes occoured when trying to unlock songs (#22)
+
+### Pokken
++ Profile saving added (loading TBA)
++ Use external STUN server for matching by default
+  + Matching still not working
+
+## 2023042300
+### Wacca
++ Time free now works properly
++ Fix reverse gate mission causing a fatal error
++ Other misc. fixes
++ Latest DB: 5
+
+### Pokken
++ Added preliminary support
+    + Nothing saves currently, but the game will boot and function properly.
+
+### Initial D Zero
++ Added preliminary support
+    + Nothing saves currently, but the game will boot and function for the most part.
+
+### Mai2
++ Added support for Festival
++ Lasted DB Version: 4
+
+### Ongeki
++ Misc fixes
++ Lasted DB Version: 4
+
+### Diva
++ Misc fixes
++ Lasted DB Version: 4
+
+### Chuni
++ Fix network encryption
++ Add `handle_remove_token_api_request` for event mode
+
+### Allnet
++ Added download order support
+    + It is up to the sysop to provide the INI file, and host the files.
+    + ONLY for use with cabs. It's not checked currently, which it's why it's default disabled
+    + YMMV, use at your own risk
++ When running develop mode, games that are not recognised will still be able to authenticate.
+
+### Database
++ Add autoupgrade command
+    + Invoke to automatically upgrade all schemas to their latest versions
+
++ `version` arg no longer required, leave it blank to update the game schema to latest if it isn't already
+
+### Misc
++ Update example nginx config file

core/adb_handlers/__init__.py

@@ -0,0 +1,6 @@
+from .base import ADBBaseRequest, ADBBaseResponse, ADBHeader, ADBHeaderException, PortalRegStatus, LogStatus, ADBStatus
+from .base import CompanyCodes, ReaderFwVer, CMD_CODE_GOODBYE, HEADER_SIZE
+from .lookup import ADBLookupRequest, ADBLookupResponse, ADBLookupExResponse
+from .campaign import ADBCampaignClearRequest, ADBCampaignClearResponse, ADBCampaignResponse, ADBOldCampaignRequest, ADBOldCampaignResponse
+from .felica import ADBFelicaLookupRequest, ADBFelicaLookupResponse, ADBFelicaLookup2Request, ADBFelicaLookup2Response
+from .log import ADBLogExRequest, ADBLogRequest, ADBStatusLogRequest, ADBLogExResponse

core/adb_handlers/base.py

@@ -0,0 +1,170 @@
+import struct
+from construct import Struct, Int16ul, Int32ul, PaddedString
+from enum import Enum
+import re
+from typing import Union, Final
+
+class LogStatus(Enum):
+    NONE = 0
+    START = 1
+    CONTINUE = 2
+    END = 3
+    OTHER = 4
+
+class PortalRegStatus(Enum):
+    NO_REG = 0
+    PORTAL = 1
+    SEGA_ID = 2
+
+class ADBStatus(Enum):
+    UNKNOWN = 0
+    GOOD = 1
+    BAD_AMIE_ID = 2
+    ALREADY_REG = 3
+    BAN_SYS_USER = 4
+    BAN_SYS = 5
+    BAN_USER = 6
+    BAN_GEN = 7
+    LOCK_SYS_USER = 8
+    LOCK_SYS = 9
+    LOCK_USER = 10
+
+class CompanyCodes(Enum):
+    NONE = 0
+    SEGA = 1
+    BAMCO = 2
+    KONAMI = 3
+    TAITO = 4
+
+class ReaderFwVer(Enum): # Newer readers use a singly byte value
+    NONE = 0
+    TN32_10 = 1
+    TN32_12 = 2
+    OTHER = 9
+    
+    def __str__(self) -> str:
+        if self == self.TN32_10:
+            return "TN32MSEC003S F/W Ver1.0"
+        elif self == self.TN32_12:
+            return "TN32MSEC003S F/W Ver1.2"
+        elif self == self.NONE:
+            return "Not Specified"
+        elif self == self.OTHER:
+            return "Unknown/Other"
+        else:
+            raise ValueError(f"Bad ReaderFwVer value {self.value}")
+    
+    @classmethod
+    def from_byte(self, byte: bytes) -> Union["ReaderFwVer", int]:
+        try:
+            i = int.from_bytes(byte, 'little')
+            try:
+                return ReaderFwVer(i)
+            except ValueError:
+                return i
+        except TypeError:
+            return 0
+
+class ADBHeaderException(Exception):
+    pass
+
+HEADER_SIZE: Final[int] = 0x20
+CMD_CODE_GOODBYE: Final[int] = 0x66
+
+# everything is LE
+class ADBHeader:
+    def __init__(self, magic: int, protocol_ver: int, cmd: int, length: int, status: int, game_id: Union[str, bytes], store_id: int, keychip_id: Union[str, bytes]) -> None:
+        self.magic = magic # u16
+        self.protocol_ver = protocol_ver # u16
+        self.cmd = cmd # u16
+        self.length = length # u16
+        try:
+            self.status = ADBStatus(status) # u16
+        except ValueError as e:
+            raise ADBHeaderException(f"Status is incorrect! {e}")
+        self.game_id = game_id # 4 char + \x00
+        self.store_id = store_id # u32
+        self.keychip_id = keychip_id# 11 char + \x00
+                
+        if type(self.game_id) == bytes:
+            self.game_id = self.game_id.decode()
+        
+        if type(self.keychip_id) == bytes:
+            self.keychip_id = self.keychip_id.decode()
+        
+        self.game_id = self.game_id.replace("\0", "")
+        self.keychip_id = self.keychip_id.replace("\0", "")
+        if self.cmd != CMD_CODE_GOODBYE: # Games for some reason send no data with goodbye
+            self.validate()
+    
+    @classmethod
+    def from_data(cls, data: bytes) -> "ADBHeader":
+        magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id = struct.unpack_from("<5H6sI12s", data)
+        head = cls(magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id)
+        
+        if head.length != len(data):
+            raise ADBHeaderException(f"Length is incorrect! Expect {head.length}, got {len(data)}")
+        
+        return head
+    
+    def validate(self) -> bool:
+        if self.magic != 0xa13e:
+            raise ADBHeaderException(f"Magic {self.magic} != 0xa13e")
+        
+        if self.protocol_ver < 0x1000:
+            raise ADBHeaderException(f"Protocol version {hex(self.protocol_ver)} is invalid!")
+
+        if re.fullmatch(r"^S[0-9A-Z]{3}[P]?$", self.game_id) is None:
+            raise ADBHeaderException(f"Game ID {self.game_id} is invalid!")
+        
+        if self.store_id == 0:
+            raise ADBHeaderException(f"Store ID cannot be 0!")
+
+        if re.fullmatch(r"^A[0-9]{2}[E|X][0-9]{2}[A-HJ-NP-Z][0-9]{4}$", self.keychip_id) is None:
+            raise ADBHeaderException(f"Keychip ID {self.keychip_id} is invalid!")
+        
+        return True
+
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "magic" / Int16ul,
+            "unknown" / Int16ul,
+            "response_code" / Int16ul,
+            "length" / Int16ul,
+            "status" / Int16ul,
+            "game_id" / PaddedString(6, 'utf_8'),
+            "store_id" / Int32ul,
+            "keychip_id" / PaddedString(12, 'utf_8'),
+        )
+
+        return resp_struct.build(dict(
+            magic=self.magic,
+            unknown=self.protocol_ver,
+            response_code=self.cmd,
+            length=self.length,
+            status=self.status.value,
+            game_id = self.game_id,
+            store_id = self.store_id,
+            keychip_id = self.keychip_id,
+        ))
+
+class ADBBaseRequest:
+    def __init__(self, data: bytes) -> None:
+        self.head = ADBHeader.from_data(data)
+
+class ADBBaseResponse:
+    def __init__(self, code: int = 0, length: int = 0x20, status: int = 1, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", protocol_ver: int = 0x3087) -> None:
+        self.head = ADBHeader(0xa13e, protocol_ver, code, length, status, game_id, store_id, keychip_id)
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, cmd: int, length: int = 0x20, status: int = 1) -> "ADBBaseResponse":
+        return cls(cmd, length, status, req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
+
+    def append_padding(self, data: bytes):
+        """Appends 0s to the end of the data until it's at the correct size"""
+        padding_size = self.head.length - len(data)
+        data += bytes(padding_size)
+        return data
+
+    def make(self) -> bytes:
+        return self.head.make()

core/adb_handlers/campaign.py

@@ -0,0 +1,132 @@
+from construct import Struct, Int16ul, Padding, Bytes, Int32ul, Int32sl
+
+from .base import *
+
+class Campaign:
+    def __init__(self) -> None:
+        self.id = 0
+        self.name = ""
+        self.announce_date = 0
+        self.start_date = 0
+        self.end_date = 0
+        self.distrib_start_date = 0
+        self.distrib_end_date = 0
+    
+    def make(self) -> bytes:
+        name_padding = bytes(128 - len(self.name))
+        return Struct(
+            "id" / Int32ul,
+            "name" / Bytes(128),
+            "announce_date" / Int32ul,
+            "start_date" / Int32ul,
+            "end_date" / Int32ul,
+            "distrib_start_date" / Int32ul,
+            "distrib_end_date" / Int32ul,
+            Padding(8),
+        ).build(dict(
+            id = self.id,
+            name = self.name.encode() + name_padding,
+            announce_date = self.announce_date,
+            start_date = self.start_date,
+            end_date = self.end_date,
+            distrib_start_date = self.distrib_start_date,
+            distrib_end_date = self.distrib_end_date,
+        ))
+
+class CampaignClear:
+    def __init__(self) -> None:
+        self.id = 0
+        self.entry_flag = 0
+        self.clear_flag = 0
+    
+    def make(self) -> bytes:
+        return Struct(
+            "id" / Int32ul,
+            "entry_flag" / Int32ul,
+            "clear_flag" / Int32ul,
+            Padding(4),
+        ).build(dict(
+            id = self.id,
+            entry_flag = self.entry_flag,
+            clear_flag = self.clear_flag,
+        ))
+
+class ADBCampaignResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0C, length: int = 0x200, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.campaigns = [Campaign(), Campaign(), Campaign()]
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self) -> bytes:
+        body = b""
+        
+        for c in self.campaigns:
+            body += c.make()
+        
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body
+
+class ADBOldCampaignRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.campaign_id = struct.unpack_from("<I", data, 0x20)
+
+class ADBOldCampaignResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0C, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.info0 = 0
+        self.info1 = 0
+        self.info2 = 0
+        self.info3 = 0
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "info0" / Int32sl,
+            "info1" / Int32sl,
+            "info2" / Int32sl,
+            "info3" / Int32sl,
+        ).build(
+            info0 = self.info0,
+            info1 = self.info1,
+            info2 = self.info2,
+            info3 = self.info3,
+        )
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+        return self.head.make() + resp_struct
+
+class ADBCampaignClearRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id = struct.unpack_from("<i", data, 0x20)
+
+class ADBCampaignClearResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0E, length: int = 0x50, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.campaign_clear_status = [CampaignClear(), CampaignClear(), CampaignClear()]
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:
+        body = b""
+        
+        for c in self.campaign_clear_status:
+            body += c.make()
+        
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/adb_handlers/felica.py

@@ -0,0 +1,84 @@
+from construct import Struct, Int32sl, Padding, Int8ub, Int16sl
+from typing import Union
+from .base import *
+
+class ADBFelicaLookupRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        idm, pmm = struct.unpack_from(">QQ", data, 0x20)
+        self.idm = hex(idm)[2:].upper()
+        self.pmm = hex(pmm)[2:].upper()
+
+class ADBFelicaLookupResponse(ADBBaseResponse):
+    def __init__(self, access_code: str = None, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x03, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.access_code = access_code if access_code is not None else "00000000000000000000"
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader, access_code: str = None) -> "ADBFelicaLookupResponse":
+        c = cls(access_code, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:        
+        resp_struct = Struct(
+            "felica_idx" / Int32ul,
+            "access_code" / Int8ub[10],
+            Padding(2)
+        ).build(dict(
+            felica_idx = 0,
+            access_code = bytes.fromhex(self.access_code)
+        ))
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+
+        return self.head.make() + resp_struct
+
+class ADBFelicaLookup2Request(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.random = struct.unpack_from("<16s", data, 0x20)[0]
+        idm, pmm = struct.unpack_from(">QQ", data, 0x30)
+        self.card_key_ver, self.write_ct, self.maca, company, fw_ver, self.dfc = struct.unpack_from("<16s16sQccH", data, 0x40)
+        self.idm = hex(idm)[2:].upper()
+        self.pmm = hex(pmm)[2:].upper()
+        self.company = CompanyCodes(int.from_bytes(company, 'little'))
+        self.fw_ver = ReaderFwVer.from_byte(fw_ver)
+
+class ADBFelicaLookup2Response(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None] = None, access_code: Union[str, None] = None, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x12, length: int = 0x130, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.access_code = access_code if access_code is not None else "00000000000000000000"
+        self.company = CompanyCodes.SEGA
+        self.portal_status = PortalRegStatus.NO_REG
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None] = None, access_code: Union[str, None] = None) -> "ADBFelicaLookup2Response":
+        c = cls(user_id, access_code, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self) -> bytes:        
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "relation1" / Int32sl,
+            "relation2" / Int32sl,
+            "access_code" / Int8ub[10],
+            "portal_status" / Int8ub,
+            "company_code" / Int8ub,
+            Padding(8),
+            "auth_key" / Int8ub[256],
+        ).build(dict(
+            user_id = self.user_id,
+            relation1 = -1, # Unsupported
+            relation2 = -1, # Unsupported
+            access_code = bytes.fromhex(self.access_code),
+            portal_status = self.portal_status.value,
+            company_code = self.company.value,
+            auth_key = [0] * 256 # Unsupported
+        ))
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+
+        return self.head.make() + resp_struct

core/adb_handlers/log.py

@@ -0,0 +1,56 @@
+from construct import Struct, Padding, Int8sl
+from typing import Final, List
+
+from .base import *
+NUM_LOGS: Final[int] = 20
+NUM_LEN_LOG_EX: Final[int] = 48
+
+class AmLogEx:
+    def __init__(self, data: bytes) -> None:
+        self.aime_id, status, self.user_id, self.credit_ct, self.bet_ct, self.won_ct, self.local_time, \
+            self.tseq, self.place_id = struct.unpack("<IIQiii4xQiI", data)
+        self.status = LogStatus(status)
+
+class ADBStatusLogRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id, status = struct.unpack_from("<II", data, 0x20)
+        self.status = LogStatus(status)
+
+class ADBLogRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id, status, self.user_id, self.credit_ct, self.bet_ct, self.won_ct = struct.unpack_from("<IIQiii", data, 0x20)
+        self.status = LogStatus(status)
+
+class ADBLogExRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.logs: List[AmLogEx] = []
+
+        for x in range(NUM_LOGS):
+            self.logs.append(AmLogEx(data[0x20 + (NUM_LEN_LOG_EX * x): 0x50 + (NUM_LEN_LOG_EX * x)]))
+        
+        self.num_logs = struct.unpack_from("<I", data, 0x03E0)[0]
+
+class ADBLogExResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", protocol_ver: int = 12423, code: int = 20, length: int = 64, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id, protocol_ver)
+
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBLogExResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
+        return c
+    
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "log_result" / Int8sl[NUM_LOGS],
+            Padding(12)
+        )
+
+        body = resp_struct.build(dict(
+            log_result = [1] * NUM_LOGS
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/adb_handlers/lookup.py

@@ -0,0 +1,81 @@
+from construct import Struct, Int32sl, Padding, Int8sl
+from typing import Union
+
+from .base import *
+
+class ADBLookupException(Exception):
+    pass
+
+class ADBLookupRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.access_code = data[0x20:0x2A].hex()
+        company_code, fw_version, self.serial_number = struct.unpack_from("<bbI", data, 0x2A)
+        
+        try:
+            self.company_code = CompanyCodes(company_code)
+        except ValueError as e:
+            raise ADBLookupException(f"Invalid company code - {e}")
+        
+        self.fw_version = ReaderFwVer.from_byte(fw_version)
+
+class ADBLookupResponse(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None], game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x06, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.portal_reg = PortalRegStatus.NO_REG
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None]) -> "ADBLookupResponse":
+        c = cls(user_id, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self):
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "portal_reg" / Int8sl,
+            Padding(11)
+        )
+
+        body = resp_struct.build(dict(
+            user_id = self.user_id,
+            portal_reg = self.portal_reg.value
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body
+
+class ADBLookupExResponse(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None], game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", 
+                code: int = 0x10, length: int = 0x130, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.portal_reg = PortalRegStatus.NO_REG
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None]) -> "ADBLookupExResponse":
+        c = cls(user_id, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self):
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "portal_reg" / Int8sl,
+            Padding(3),
+            "auth_key" / Int8sl[256],
+            "relation1" / Int32sl,
+            "relation2" / Int32sl,
+        )
+
+        body = resp_struct.build(dict(
+            user_id = self.user_id,
+            portal_reg = self.portal_reg.value,
+            auth_key = [0] * 256,
+            relation1 = -1,
+            relation2 = -1
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/aimedb.py

@@ -2,27 +2,17 @@ from twisted.internet.protocol import Factory, Protocol
 import logging, coloredlogs
 from Crypto.Cipher import AES
 import struct
-from typing import Dict, Any
+from typing import Dict, Tuple, Callable, Union
+from typing_extensions import Final
 from logging.handlers import TimedRotatingFileHandler
 
 from core.config import CoreConfig
 from core.data import Data
+from .adb_handlers import *
 
 
 class AimedbProtocol(Protocol):
-    AIMEDB_RESPONSE_CODES = {
+    request_list: Dict[int, Tuple[Callable[[bytes, int], Union[ADBBaseResponse, bytes]], int, str]] = {}
-        "felica_lookup": 0x03,
-        "lookup": 0x06,
-        "log": 0x0A,
-        "campaign": 0x0C,
-        "touch": 0x0E,
-        "lookup2": 0x10,
-        "felica_lookup2": 0x12,
-        "log2": 0x14,
-        "hello": 0x65,
-    }
-
-    request_list: Dict[int, Any] = {}
 
     def __init__(self, core_cfg: CoreConfig) -> None:
         self.logger = logging.getLogger("aimedb")
@@ -32,16 +22,27 @@ class AimedbProtocol(Protocol):
             self.logger.error("!!!KEY NOT SET!!!")
             exit(1)
 
-        self.request_list[0x01] = self.handle_felica_lookup
+        self.register_handler(0x01, 0x03, self.handle_felica_lookup, 'felica_lookup')
-        self.request_list[0x04] = self.handle_lookup
+        self.register_handler(0x02, 0x03, self.handle_felica_register, 'felica_register')
-        self.request_list[0x05] = self.handle_register
+
-        self.request_list[0x09] = self.handle_log
+        self.register_handler(0x04, 0x06, self.handle_lookup, 'lookup')
-        self.request_list[0x0B] = self.handle_campaign
+        self.register_handler(0x05, 0x06, self.handle_register, 'register')
-        self.request_list[0x0D] = self.handle_touch
+
-        self.request_list[0x0F] = self.handle_lookup2
+        self.register_handler(0x07, 0x08, self.handle_status_log, 'status_log')
-        self.request_list[0x11] = self.handle_felica_lookup2
+        self.register_handler(0x09, 0x0A, self.handle_log, 'aime_log')       
-        self.request_list[0x13] = self.handle_log2
+
-        self.request_list[0x64] = self.handle_hello
+        self.register_handler(0x0B, 0x0C, self.handle_campaign, 'campaign')
+        self.register_handler(0x0D, 0x0E, self.handle_campaign_clear, 'campaign_clear')
+
+        self.register_handler(0x0F, 0x10, self.handle_lookup_ex, 'lookup_ex')
+        self.register_handler(0x11, 0x12, self.handle_felica_lookup_ex, 'felica_lookup_ex')
+
+        self.register_handler(0x13, 0x14, self.handle_log_ex, 'aime_log_ex')
+        self.register_handler(0x64, 0x65, self.handle_hello, 'hello')
+        self.register_handler(0x66, 0, self.handle_goodbye, 'goodbye')
+    
+    def register_handler(self, cmd: int, resp:int, handler: Callable[[bytes, int], Union[ADBBaseResponse, bytes]], name: str) -> None:
+        self.request_list[cmd] = (handler, resp, name)
 
     def append_padding(self, data: bytes):
         """Appends 0s to the end of the data until it's at the correct size"""
@@ -63,202 +64,233 @@ class AimedbProtocol(Protocol):
 
         try:
             decrypted = cipher.decrypt(data)
-        except:
+        
-            self.logger.error(f"Failed to decrypt {data.hex()}")
+        except Exception as e:
+            self.logger.error(f"Failed to decrypt {data.hex()} because {e}")
             return None
 
         self.logger.debug(f"{self.transport.getPeer().host} wrote {decrypted.hex()}")
 
-        if not decrypted[1] == 0xA1 and not decrypted[0] == 0x3E:
+        try:
-            self.logger.error(f"Bad magic")
+            head = ADBHeader.from_data(decrypted)
-            return None
         
-        req_code = decrypted[4]
+        except ADBHeaderException as e:
+            self.logger.error(f"Error parsing ADB header: {e}")
+            try:    
+                encrypted = cipher.encrypt(ADBBaseResponse().make())
+                self.transport.write(encrypted)
+
+            except Exception as e:
+                self.logger.error(f"Failed to encrypt default response because {e}")
             
-        if req_code == 0x66:
-            self.logger.info(f"goodbye from {self.transport.getPeer().host}")
-            self.transport.loseConnection()
             return
 
+        if head.keychip_id == "ABCD1234567" or head.store_id == 0xfff0:
+            self.logger.warning(f"Request from uninitialized AMLib: {vars(head)}")
+
+        handler, resp_code, name = self.request_list.get(head.cmd, (self.handle_default, None, 'default'))
+
+        if resp_code is None:
+            self.logger.warning(f"No handler for cmd {hex(head.cmd)}")
+        
+        elif resp_code > 0:
+            self.logger.info(f"{name} from {head.keychip_id} ({head.game_id}) @ {self.transport.getPeer().host}")
+        
+        resp = handler(decrypted, resp_code)
+
+        if type(resp) == ADBBaseResponse or issubclass(type(resp), ADBBaseResponse):
+            resp_bytes = resp.make()
+            if len(resp_bytes) != resp.head.length:
+                resp_bytes = self.append_padding(resp_bytes)
+
+        elif type(resp) == bytes:
+            resp_bytes = resp
+        
+        elif resp is None: # Nothing to send, probably a goodbye
+            return
+        
+        else:
+            raise TypeError(f"Unsupported type returned by ADB handler for {name}: {type(resp)}")
+
         try:    
-            resp = self.request_list[req_code](decrypted)
+            encrypted = cipher.encrypt(resp_bytes)
-            encrypted = cipher.encrypt(resp)
+            self.logger.debug(f"Response {resp_bytes.hex()}")
-            self.logger.debug(f"Response {resp.hex()}")
             self.transport.write(encrypted)
 
-        except KeyError:
+        except Exception as e:
-            self.logger.error(f"Unknown command code {hex(req_code)}")
+            self.logger.error(f"Failed to encrypt {resp_bytes.hex()} because {e}")
-            return None
         
-        except ValueError as e:
+    def handle_default(self, data: bytes, resp_code: int, length: int = 0x20) -> ADBBaseResponse:
-            self.logger.error(f"Failed to encrypt {resp.hex()} because {e}")
+        req = ADBHeader.from_data(data)
-            return None
+        return ADBBaseResponse(resp_code, length, 1, req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
 
-    def handle_campaign(self, data: bytes) -> bytes:
+    def handle_hello(self, data: bytes, resp_code: int) -> ADBBaseResponse:
-        self.logger.info(f"campaign from {self.transport.getPeer().host}")
+        return self.handle_default(data, resp_code)
-        ret = struct.pack(
-            "<5H",
-            0xA13E,
-            0x3087,
-            self.AIMEDB_RESPONSE_CODES["campaign"],
-            0x0200,
-            0x0001,
-        )
-        return self.append_padding(ret)
 
-    def handle_hello(self, data: bytes) -> bytes:
+    def handle_campaign(self, data: bytes, resp_code: int) -> ADBBaseResponse:
-        self.logger.info(f"hello from {self.transport.getPeer().host}")
+        h = ADBHeader.from_data(data)
-        ret = struct.pack(
+        if h.protocol_ver >= 0x3030:
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["hello"], 0x0020, 0x0001
+            req = h
-        )
+            resp = ADBCampaignResponse.from_req(req)
-        return self.append_padding(ret)
 
-    def handle_lookup(self, data: bytes) -> bytes:
-        luid = data[0x20:0x2A].hex()
-        user_id = self.data.card.get_user_id_from_card(access_code=luid)
-
-        if user_id is None:
-            user_id = -1
-
-        self.logger.info(
-            f"lookup from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}"
-        )
-
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["lookup"], 0x0130, 0x0001
-        )
-        ret += bytes(0x20 - len(ret))
-
-        if user_id is None:
-            ret += struct.pack("<iH", -1, 0)
         else:
-            ret += struct.pack("<l", user_id)
+            req = ADBOldCampaignRequest(data)
-        return self.append_padding(ret)
             
-    def handle_lookup2(self, data: bytes) -> bytes:
+            self.logger.info(f"Legacy campaign request for campaign {req.campaign_id} (protocol version {hex(h.protocol_ver)})")
-        self.logger.info(f"lookup2")
+            resp = ADBOldCampaignResponse.from_req(req.head)
         
-        ret = bytearray(self.handle_lookup(data))
+        # We don't currently support campaigns
-        ret[4] = self.AIMEDB_RESPONSE_CODES["lookup2"]
+        return resp
 
-        return bytes(ret)
+    def handle_lookup(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        req = ADBLookupRequest(data)
+        user_id = self.data.card.get_user_id_from_card(req.access_code)
+
+        ret = ADBLookupResponse.from_req(req.head, user_id)
         
-    def handle_felica_lookup(self, data: bytes) -> bytes:
-        idm = data[0x20:0x28].hex()
-        pmm = data[0x28:0x30].hex()
-        access_code = self.data.card.to_access_code(idm)
         self.logger.info(
-            f"felica_lookup from {self.transport.getPeer().host}: idm {idm} pmm {pmm} -> access_code {access_code}"
+            f"access_code {req.access_code} -> user_id {ret.user_id}"
         )
+        return ret
 
-        ret = struct.pack(
+    def handle_lookup_ex(self, data: bytes, resp_code: int) -> ADBBaseResponse:
-            "<5H",
+        req = ADBLookupRequest(data)
-            0xA13E,
+        user_id = self.data.card.get_user_id_from_card(req.access_code)
-            0x3087,
+
-            self.AIMEDB_RESPONSE_CODES["felica_lookup"],
+        ret = ADBLookupExResponse.from_req(req.head, user_id)
-            0x0030,
+        
-            0x0001,
+        self.logger.info(
+            f"access_code {req.access_code} -> user_id {ret.user_id}"
         )
-        ret += bytes(26)
+        return ret
-        ret += bytes.fromhex(access_code)
 
-        return self.append_padding(ret)
+    def handle_felica_lookup(self, data: bytes, resp_code: int) -> bytes:
+        """
+        On official, I think a card has to be registered for this to actually work, but 
+        I'm making the executive decision to not implement that and just kick back our
+        faux generated access code. The real felica IDm -> access code conversion is done
+        on the ADB server, which we do not and will not ever have access to. Because we can
+        assure that all IDms will be unique, this basic 0-padded hex -> int conversion will
+        be fine.
+        """
+        req = ADBFelicaLookupRequest(data)
+        ac = self.data.card.to_access_code(req.idm)
+        self.logger.info(
+            f"idm {req.idm} ipm {req.pmm} -> access_code {ac}"
+        )
+        return ADBFelicaLookupResponse.from_req(req.head, ac)
 
-    def handle_felica_lookup2(self, data: bytes) -> bytes:
+    def handle_felica_register(self, data: bytes, resp_code: int) -> bytes:
-        idm = data[0x30:0x38].hex()
+        """
-        pmm = data[0x38:0x40].hex()
+        I've never seen this used.
-        access_code = self.data.card.to_access_code(idm)
+        """
+        req = ADBFelicaLookupRequest(data)
+        ac = self.data.card.to_access_code(req.idm)
+        
+        if self.config.server.allow_user_registration:
+            user_id = self.data.user.create_user()
+
+            if user_id is None:
+                self.logger.error("Failed to register user!")
+                user_id = -1
+
+            else:
+                card_id = self.data.card.create_card(user_id, ac)
+
+                if card_id is None:
+                    self.logger.error("Failed to register card!")
+                    user_id = -1
+
+            self.logger.info(
+                f"Register access code {ac} (IDm: {req.idm} PMm: {req.pmm}) -> user_id {user_id}"
+            )
+
+        else:
+            self.logger.info(
+                f"Registration blocked!: access code {ac} (IDm: {req.idm} PMm: {req.pmm})"
+            )
+
+        return ADBFelicaLookupResponse.from_req(req.head, ac)
+
+    def handle_felica_lookup_ex(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBFelicaLookup2Request(data)
+        access_code = self.data.card.to_access_code(req.idm)
         user_id = self.data.card.get_user_id_from_card(access_code=access_code)
 
         if user_id is None:
             user_id = -1
 
         self.logger.info(
-            f"felica_lookup2 from {self.transport.getPeer().host}: idm {idm} ipm {pmm} -> access_code {access_code} user_id {user_id}"
+            f"idm {req.idm} ipm {req.pmm} -> access_code {access_code} user_id {user_id}"
         )
 
-        ret = struct.pack(
+        return ADBFelicaLookup2Response.from_req(req.head, user_id, access_code)
-            "<5H",
-            0xA13E,
-            0x3087,
-            self.AIMEDB_RESPONSE_CODES["felica_lookup2"],
-            0x0140,
-            0x0001,
-        )
-        ret += bytes(22)
-        ret += struct.pack("<lq", user_id, -1)  # first -1 is ext_id, 3rd is access code
-        ret += bytes.fromhex(access_code)
-        ret += struct.pack("<l", 1)
 
-        return self.append_padding(ret)
+    def handle_campaign_clear(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        req = ADBCampaignClearRequest(data)
 
-    def handle_touch(self, data: bytes) -> bytes:
+        resp = ADBCampaignClearResponse.from_req(req.head)
-        self.logger.info(f"touch from {self.transport.getPeer().host}")
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["touch"], 0x0050, 0x0001
-        )
-        ret += bytes(5)
-        ret += struct.pack("<3H", 0x6F, 0, 1)
 
-        return self.append_padding(ret)
+        # We don't support campaign stuff
+        return resp
+
+    def handle_register(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBLookupRequest(data)
+        user_id = -1
         
-    def handle_register(self, data: bytes) -> bytes:
-        luid = data[0x20:0x2A].hex()
         if self.config.server.allow_user_registration:
             user_id = self.data.user.create_user()
 
             if user_id is None:
-                user_id = -1
                 self.logger.error("Failed to register user!")
+                user_id = -1
 
             else:
-                card_id = self.data.card.create_card(user_id, luid)
+                card_id = self.data.card.create_card(user_id, req.access_code)
 
                 if card_id is None:
-                    user_id = -1
                     self.logger.error("Failed to register card!")
+                    user_id = -1
 
             self.logger.info(
-                f"register from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}"
+                f"Register access code {req.access_code} -> user_id {user_id}"
             )
 
         else:
             self.logger.info(
-                f"register from {self.transport.getPeer().host} blocked!: luid {luid}"
+                f"Registration blocked!: access code {req.access_code}"
             )
-            user_id = -1
 
-        ret = struct.pack(
+        resp = ADBLookupResponse.from_req(req.head, user_id)
-            "<5H",
+        if resp.user_id <= 0:
-            0xA13E,
+            resp.head.status = ADBStatus.BAN_SYS # Closest we can get to a "You cannot register"
-            0x3087,
-            self.AIMEDB_RESPONSE_CODES["lookup"],
-            0x0030,
-            0x0001 if user_id > -1 else 0,
-        )
-        ret += bytes(0x20 - len(ret))
-        ret += struct.pack("<l", user_id)
 
-        return self.append_padding(ret)
+        return resp
 
-    def handle_log(self, data: bytes) -> bytes:
+    # TODO: Save these in some capacity, as deemed relevant
-        # TODO: Save aimedb logs
+    def handle_status_log(self, data: bytes, resp_code: int) -> bytes:
-        self.logger.info(f"log from {self.transport.getPeer().host}")
+        req = ADBStatusLogRequest(data)
-        ret = struct.pack(
+        self.logger.info(f"User {req.aime_id} logged {req.status.name} event")
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["log"], 0x0020, 0x0001
+        return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
-        )
-        return self.append_padding(ret)
 
-    def handle_log2(self, data: bytes) -> bytes:
+    def handle_log(self, data: bytes, resp_code: int) -> bytes:
-        self.logger.info(f"log2 from {self.transport.getPeer().host}")
+        req = ADBLogRequest(data)
-        ret = struct.pack(
+        self.logger.info(f"User {req.aime_id} logged {req.status.name} event, credit_ct: {req.credit_ct} bet_ct: {req.bet_ct} won_ct: {req.won_ct}")
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["log2"], 0x0040, 0x0001
+        return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
-        )
-        ret += bytes(22)
-        ret += struct.pack("H", 1)
 
-        return self.append_padding(ret)
+    def handle_log_ex(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBLogExRequest(data)
+        strs = []
+        self.logger.info(f"Recieved {req.num_logs} or {len(req.logs)} logs")
         
+        for x in range(req.num_logs):
+            self.logger.debug(f"User {req.logs[x].aime_id} logged {req.logs[x].status.name} event, credit_ct: {req.logs[x].credit_ct} bet_ct: {req.logs[x].bet_ct} won_ct: {req.logs[x].won_ct}")
+        return ADBLogExResponse.from_req(req.head)
+
+    def handle_goodbye(self, data: bytes, resp_code: int) -> None:
+        self.logger.info(f"goodbye from {self.transport.getPeer().host}")
+        self.transport.loseConnection()
+        return
 
 class AimedbFactory(Factory):
     protocol = AimedbProtocol

core/allnet.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Any, Optional, Tuple
+from typing import Dict, List, Any, Optional, Tuple, Union, Final
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
 from twisted.web.http import Request
@@ -6,16 +6,32 @@ from datetime import datetime
 import pytz
 import base64
 import zlib
+import json
+from enum import Enum
 from Crypto.PublicKey import RSA
 from Crypto.Hash import SHA
 from Crypto.Signature import PKCS1_v1_5
 from time import strptime
+from os import path
+import urllib.parse
+import math
 
 from core.config import CoreConfig
 from core.utils import Utils
 from core.data import Data
 from core.const import *
 
+BILLING_DT_FORMAT: Final[str] = "%Y%m%d%H%M%S"
+
+class DLIMG_TYPE(Enum):
+    app = 0
+    opt = 1
+
+class ALLNET_STAT(Enum):
+    ok = 0
+    bad_game = -1
+    bad_machine = -2
+    bad_shop = -3
 
 class AllnetServlet:
     def __init__(self, core_cfg: CoreConfig, cfg_folder: str):
@@ -55,7 +71,7 @@ class AllnetServlet:
             self.logger.error("No games detected!")
 
         for _, mod in plugins.items():
-            if hasattr(mod.index, "get_allnet_info"):
+            if hasattr(mod, "index") and hasattr(mod.index, "get_allnet_info"):
                 for code in mod.game_codes:
                     enabled, uri, host = mod.index.get_allnet_info(
                         code, self.config, self.config_folder
@@ -70,15 +86,23 @@ class AllnetServlet:
 
     def handle_poweron(self, request: Request, _: Dict):
         request_ip = Utils.get_ip_addr(request)
+        pragma_header = request.getHeader('Pragma')
+        is_dfi = pragma_header is not None and pragma_header == "DFI"
+        
         try:
-            req_dict = self.allnet_req_to_dict(request.content.getvalue())
+            if is_dfi:
+                req_urlencode = self.from_dfi(request.content.getvalue())
+            else:
+                req_urlencode = request.content.getvalue().decode()
+
+            req_dict = self.allnet_req_to_dict(req_urlencode)
             if req_dict is None:
                 raise AllnetRequestException()
 
             req = AllnetPowerOnRequest(req_dict[0])
             # Validate the request. Currently we only validate the fields we plan on using
 
-            if not req.game_id or not req.ver or not req.serial or not req.ip:
+            if not req.game_id or not req.ver or not req.serial or not req.ip or not req.firm_ver or not req.boot_ver:
                 raise AllnetRequestException(
                     f"Bad auth request params from {request_ip} - {vars(req)}"
                 )
@@ -88,30 +112,14 @@ class AllnetServlet:
                 self.logger.error(e)
             return b""
 
-        if req.format_ver == "3":
+        if req.format_ver == 3:
             resp = AllnetPowerOnResponse3(req.token)
-        else:
+        elif req.format_ver == 2:
             resp = AllnetPowerOnResponse2()
+        else:
+            resp = AllnetPowerOnResponse()
 
         self.logger.debug(f"Allnet request: {vars(req)}")
-        if req.game_id not in self.uri_registry:
-            if not self.config.server.is_develop:
-                msg = f"Unrecognised game {req.game_id} attempted allnet auth from {request_ip}."
-                self.data.base.log_event(
-                    "allnet", "ALLNET_AUTH_UNKNOWN_GAME", logging.WARN, msg
-                )
-                self.logger.warn(msg)
-
-                resp.stat = 0
-                return self.dict_to_http_form_string([vars(resp)])
-            
-            else:
-                self.logger.info(f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}")
-                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/{req.game_id}/{req.ver.replace('.', '')}/"
-                resp.host = f"{self.config.title.hostname}:{self.config.title.port}"
-                return self.dict_to_http_form_string([vars(resp)])
-
-        resp.uri, resp.host = self.uri_registry[req.game_id]
 
         machine = self.data.arcade.get_machine(req.serial)        
         if machine is None and not self.config.server.allow_unregistered_serials:
@@ -119,13 +127,38 @@ class AllnetServlet:
             self.data.base.log_event(
                 "allnet", "ALLNET_AUTH_UNKNOWN_SERIAL", logging.WARN, msg
             )
-            self.logger.warn(msg)
+            self.logger.warning(msg)
 
-            resp.stat = 0
+            resp.stat = ALLNET_STAT.bad_machine.value
-            return self.dict_to_http_form_string([vars(resp)])
+            resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+            return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
 
         if machine is not None:
             arcade = self.data.arcade.get_arcade(machine["arcade"])
+            if self.config.server.check_arcade_ip:
+                if arcade["ip"] and arcade["ip"] is not None and arcade["ip"] != req.ip:
+                    msg = f"Serial {req.serial} attempted allnet auth from bad IP {req.ip} (expected {arcade['ip']})."
+                    self.data.base.log_event(
+                        "allnet", "ALLNET_AUTH_BAD_IP", logging.ERROR, msg
+                    )
+                    self.logger.warning(msg)
+
+                    resp.stat = ALLNET_STAT.bad_shop.value
+                    resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                    return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+                
+                elif not arcade["ip"] or arcade["ip"] is None and self.config.server.strict_ip_checking:
+                    msg = f"Serial {req.serial} attempted allnet auth from bad IP {req.ip}, but arcade {arcade['id']} has no IP set! (strict checking enabled)."
+                    self.data.base.log_event(
+                        "allnet", "ALLNET_AUTH_NO_SHOP_IP", logging.ERROR, msg
+                    )
+                    self.logger.warning(msg)
+
+                    resp.stat = ALLNET_STAT.bad_shop.value
+                    resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                    return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+
+
             country = (
                 arcade["country"] if machine["country"] is None else machine["country"]
             )
@@ -157,6 +190,33 @@ class AllnetServlet:
                 arcade["timezone"] if arcade["timezone"] is not None else "+0900"
             )
         
+        if req.game_id not in self.uri_registry:
+            if not self.config.server.is_develop:
+                msg = f"Unrecognised game {req.game_id} attempted allnet auth from {request_ip}."
+                self.data.base.log_event(
+                    "allnet", "ALLNET_AUTH_UNKNOWN_GAME", logging.WARN, msg
+                )
+                self.logger.warning(msg)
+
+                resp.stat = ALLNET_STAT.bad_game.value
+                resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+
+            else:
+                self.logger.info(
+                    f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}"
+                )
+                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/{req.game_id}/{req.ver.replace('.', '')}/"
+                resp.host = f"{self.config.title.hostname}:{self.config.title.port}"
+                
+                resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
+                
+                self.logger.debug(f"Allnet response: {resp_str}")
+                return (resp_str + "\n").encode("utf-8")
+
+        resp.uri, resp.host = self.uri_registry[req.game_id]
+
         int_ver = req.ver.replace(".", "")
         resp.uri = resp.uri.replace("$v", int_ver)
         resp.host = resp.host.replace("$v", int_ver)
@@ -164,14 +224,30 @@ class AllnetServlet:
         msg = f"{req.serial} authenticated from {request_ip}: {req.game_id} v{req.ver}"
         self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
         self.logger.info(msg)
-        self.logger.debug(f"Allnet response: {vars(resp)}")
 
-        return self.dict_to_http_form_string([vars(resp)]).encode("utf-8")
+        resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+        resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
+        self.logger.debug(f"Allnet response: {resp_dict}")        
+        resp_str += "\n"
+
+        """if is_dfi:
+            request.responseHeaders.addRawHeader('Pragma', 'DFI')
+            return self.to_dfi(resp_str)"""
+
+        return resp_str.encode("utf-8")
 
     def handle_dlorder(self, request: Request, _: Dict):
         request_ip = Utils.get_ip_addr(request)
+        pragma_header = request.getHeader('Pragma')
+        is_dfi = pragma_header is not None and pragma_header == "DFI"
+        
         try:
-            req_dict = self.allnet_req_to_dict(request.content.getvalue())
+            if is_dfi:
+                req_urlencode = self.from_dfi(request.content.getvalue())
+            else:
+                req_urlencode = request.content.getvalue().decode()
+
+            req_dict = self.allnet_req_to_dict(req_urlencode)
             if req_dict is None:
                 raise AllnetRequestException()
 
@@ -188,18 +264,115 @@ class AllnetServlet:
                 self.logger.error(e)
             return b""
 
-        self.logger.info(f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}")
+        self.logger.info(
-        resp = AllnetDownloadOrderResponse()
+            f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}"
+        )
+        resp = AllnetDownloadOrderResponse(serial=req.serial)
 
-        if not self.config.allnet.allow_online_updates:
+        if (
-            return self.dict_to_http_form_string([vars(resp)])
+            not self.config.allnet.allow_online_updates
+            or not self.config.allnet.update_cfg_folder
+        ):
+            return urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
 
-        else:  # TODO: Actual dlorder response
+        else:  # TODO: Keychip check
-            return self.dict_to_http_form_string([vars(resp)])
+            if path.exists(
+                f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
+            ):
+                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
+
+            if path.exists(
+                f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
+            ):
+                resp.uri += f"|http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
+
+            self.logger.debug(f"Sending download uri {resp.uri}")
+            self.data.base.log_event("allnet", "DLORDER_REQ_SUCCESS", logging.INFO, f"{Utils.get_ip_addr(request)} requested DL Order for {req.serial} {req.game_id} v{req.ver}")
+
+            res_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
+            """if is_dfi:
+                request.responseHeaders.addRawHeader('Pragma', 'DFI')
+                return self.to_dfi(res_str)"""
+
+            return res_str
+            
+
+    def handle_dlorder_ini(self, request: Request, match: Dict) -> bytes:
+        if "file" not in match:
+            return b""
+
+        req_file = match["file"].replace("%0A", "")
+
+        if path.exists(f"{self.config.allnet.update_cfg_folder}/{req_file}"):
+            self.logger.info(f"Request for DL INI file {req_file} from {Utils.get_ip_addr(request)} successful")
+            self.data.base.log_event("allnet", "DLORDER_INI_SENT", logging.INFO, f"{Utils.get_ip_addr(request)} successfully recieved {req_file}")
+            
+            return open(
+                f"{self.config.allnet.update_cfg_folder}/{req_file}", "rb"
+            ).read()
+
+        self.logger.info(f"DL INI File {req_file} not found")
+        return b""
+
+    def handle_dlorder_report(self, request: Request, match: Dict) -> bytes:
+        req_raw = request.content.getvalue()
+        try:
+            req_dict: Dict = json.loads(req_raw)
+        except Exception as e:
+            self.logger.warning(f"Failed to parse DL Report: {e}")
+            return "NG"
+        
+        dl_data_type = DLIMG_TYPE.app
+        dl_data = req_dict.get("appimage", {})
+        
+        if dl_data is None or not dl_data:
+            dl_data_type = DLIMG_TYPE.opt
+            dl_data = req_dict.get("optimage", {})
+        
+        if dl_data is None or not dl_data:
+            self.logger.warning(f"Failed to parse DL Report: Invalid format - contains neither appimage nor optimage")
+            return "NG"
+
+        dl_report_data = DLReport(dl_data, dl_data_type)
+
+        if not dl_report_data.validate():
+            self.logger.warning(f"Failed to parse DL Report: Invalid format - {dl_report_data.err}")
+            return "NG"
+
+        return "OK"
+
+    def handle_loaderstaterecorder(self, request: Request, match: Dict) -> bytes:
+        req_data = request.content.getvalue()
+        sections = req_data.decode("utf-8").split("\r\n")
+        
+        req_dict = dict(urllib.parse.parse_qsl(sections[0]))
+
+        serial: Union[str, None] = req_dict.get("serial", None)
+        num_files_to_dl: Union[str, None] = req_dict.get("nb_ftd", None)
+        num_files_dld: Union[str, None] = req_dict.get("nb_dld", None)
+        dl_state: Union[str, None] = req_dict.get("dld_st", None)
+        ip = Utils.get_ip_addr(request)
+
+        if serial is None or num_files_dld is None or num_files_to_dl is None or dl_state is None:
+            return "NG".encode()
+
+        self.logger.info(f"LoaderStateRecorder Request from {ip} {serial}: {num_files_dld}/{num_files_to_dl} Files download (State: {dl_state})")
+        return "OK".encode()
+    
+    def handle_alive(self, request: Request, match: Dict) -> bytes:
+        return "OK".encode()
 
     def handle_billing_request(self, request: Request, _: Dict):
-        req_dict = self.billing_req_to_dict(request.content.getvalue())
+        req_raw = request.content.getvalue()
+        
+        if request.getHeader('Content-Type') == "application/octet-stream":
+            req_unzip = zlib.decompressobj(-zlib.MAX_WBITS).decompress(req_raw)
+        else:
+            req_unzip = req_raw
+        
+        req_dict = self.billing_req_to_dict(req_unzip)
         request_ip = Utils.get_ip_addr(request)
+        
         if req_dict is None:
             self.logger.error(f"Failed to parse request {request.content.getvalue()}")
             return b""
@@ -209,41 +382,60 @@ class AllnetServlet:
         rsa = RSA.import_key(open(self.config.billing.signing_key, "rb").read())
         signer = PKCS1_v1_5.new(rsa)
         digest = SHA.new()
+        traces: List[TraceData] = []
 
-        kc_playlimit = int(req_dict[0]["playlimit"])
+        try:
-        kc_nearfull = int(req_dict[0]["nearfull"])
+            for x in range(len(req_dict)):
-        kc_billigtype = int(req_dict[0]["billingtype"])
+                if not req_dict[x]:
-        kc_playcount = int(req_dict[0]["playcnt"])
+                    continue
-        kc_serial: str = req_dict[0]["keychipid"]
-        kc_game: str = req_dict[0]["gameid"]
-        kc_date = strptime(req_dict[0]["date"], "%Y%m%d%H%M%S")
-        kc_serial_bytes = kc_serial.encode()
                 
-        machine = self.data.arcade.get_machine(kc_serial)
+                if x == 0:                    
+                    req = BillingInfo(req_dict[x])
+                    continue
+
+                tmp = TraceData(req_dict[x])
+                if tmp.trace_type == TraceDataType.CHARGE:
+                    tmp = TraceDataCharge(req_dict[x])
+                elif tmp.trace_type == TraceDataType.EVENT:
+                    tmp = TraceDataEvent(req_dict[x])
+                elif tmp.trace_type == TraceDataType.CREDIT:
+                    tmp = TraceDataCredit(req_dict[x])
+                
+                traces.append(tmp)
+
+            kc_serial_bytes = req.keychipid.encode()
+        
+        except KeyError as e:
+            self.logger.error(f"Billing request failed to parse: {e}")
+            return f"result=5&linelimit=&message=field is missing or formatting is incorrect\r\n".encode()
+
+        machine = self.data.arcade.get_machine(req.keychipid)
         if machine is None and not self.config.server.allow_unregistered_serials:
-            msg = f"Unrecognised serial {kc_serial} attempted billing checkin from {request_ip} for game {kc_game}."
+            msg = f"Unrecognised serial {req.keychipid} attempted billing checkin from {request_ip} for {req.gameid} v{req.gamever}."
             self.data.base.log_event(
                 "allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg
             )
-            self.logger.warn(msg)
+            self.logger.warning(msg)
 
-            resp = BillingResponse("", "", "", "")
+            return f"result=1&requestno={req.requestno}&message=Keychip Serial bad\r\n".encode()
-            resp.result = "1"
-            return self.dict_to_http_form_string([vars(resp)])
 
         msg = (
-            f"Billing checkin from {request_ip}: game {kc_game} keychip {kc_serial} playcount "
+            f"Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount "
-            f"{kc_playcount} billing_type {kc_billigtype} nearfull {kc_nearfull} playlimit {kc_playlimit}"
+            f"{req.playcnt} billing_type {req.billingtype.name} nearfull {req.nearfull} playlimit {req.playlimit}"
         )
         self.logger.info(msg)
         self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, msg)
+        if req.traceleft > 0:
+            self.logger.warn(f"{req.traceleft} unsent tracelogs")
+        kc_playlimit = req.playlimit
+        kc_nearfull = req.nearfull
 
-        while kc_playcount > kc_playlimit:
+        while req.playcnt > req.playlimit:
             kc_playlimit += 1024
             kc_nearfull += 1024
 
         playlimit = kc_playlimit
-        nearfull = kc_nearfull + (kc_billigtype * 0x00010000)
+        nearfull = kc_nearfull + (req.billingtype.value * 0x00010000)
 
         digest.update(playlimit.to_bytes(4, "little") + kc_serial_bytes)
         playlimit_sig = signer.sign(digest).hex()
@@ -254,44 +446,33 @@ class AllnetServlet:
 
         # TODO: playhistory
 
-        resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig)
+        #resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig)
+        resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig, req.requestno, req.protocolver)
 
-        resp_str = self.dict_to_http_form_string([vars(resp)], True)
+        resp_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\r\n"
-        if resp_str is None:
-            self.logger.error(f"Failed to parse response {vars(resp)}")
 
         self.logger.debug(f"response {vars(resp)}")
+        if req.traceleft > 0:
+            self.logger.info(f"Requesting 20 more of {req.traceleft} unsent tracelogs")
+            return f"result=6&waittime=0&linelimit=20\r\n".encode()
+
         return resp_str.encode("utf-8")
 
     def handle_naomitest(self, request: Request, _: Dict) -> bytes:
         self.logger.info(f"Ping from {Utils.get_ip_addr(request)}")
         return b"naomi ok"
 
-    def kvp_to_dict(self, kvp: List[str]) -> List[Dict[str, Any]]:
-        ret: List[Dict[str, Any]] = []
-        for x in kvp:
-            items = x.split("&")
-            tmp = {}
-
-            for item in items:
-                kvp = item.split("=")
-                if len(kvp) == 2:
-                    tmp[kvp[0]] = kvp[1]
-
-            ret.append(tmp)
-
-        return ret
-
     def billing_req_to_dict(self, data: bytes):
         """
         Parses an billing request string into a python dictionary
         """
         try:
-            decomp = zlib.decompressobj(-zlib.MAX_WBITS)
+            sections = data.decode("ascii").split("\r\n")
-            unzipped = decomp.decompress(data)
-            sections = unzipped.decode("ascii").split("\r\n")
 
-            return self.kvp_to_dict(sections)
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
 
         except Exception as e:
             self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
@@ -302,111 +483,94 @@ class AllnetServlet:
         Parses an allnet request string into a python dictionary
         """
         try:
-            zipped = base64.b64decode(data)
+            sections = data.split("\r\n")
-            unzipped = zlib.decompress(zipped)
-            sections = unzipped.decode("utf-8").split("\r\n")
 
-            return self.kvp_to_dict(sections)
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
 
         except Exception as e:
             self.logger.error(f"allnet_req_to_dict: {e} while parsing {data}")
             return None
 
-    def dict_to_http_form_string(
+    def from_dfi(self, data: bytes) -> str:
-        self,
+        zipped = base64.b64decode(data)
-        data: List[Dict[str, Any]],
+        unzipped = zlib.decompress(zipped)
-        crlf: bool = False,
+        return unzipped.decode("utf-8")
-        trailing_newline: bool = True,
-    ) -> Optional[str]:
-        """
-        Takes a python dictionary and parses it into an allnet response string
-        """
-        try:
-            urlencode = ""
-            for item in data:
-                for k, v in item.items():
-                    urlencode += f"{k}={v}&"
 
-                if crlf:
+    def to_dfi(self, data: str) -> bytes:
-                    urlencode = urlencode[:-1] + "\r\n"
+        unzipped = data.encode('utf-8')
-                else:
+        zipped = zlib.compress(unzipped)
-                    urlencode = urlencode[:-1] + "\n"
+        return base64.b64encode(zipped)
-
-            if not trailing_newline:
-                if crlf:
-                    urlencode = urlencode[:-2]
-                else:
-                    urlencode = urlencode[:-1]
-
-            return urlencode
-
-        except Exception as e:
-            self.logger.error(f"dict_to_http_form_string: {e} while parsing {data}")
-            return None
 
 
 class AllnetPowerOnRequest:
     def __init__(self, req: Dict) -> None:
         if req is None:
             raise AllnetRequestException("Request processing failed")
-        self.game_id: str = req.get("game_id", "")
+        self.game_id: str = req.get("game_id", None)
-        self.ver: str = req.get("ver", "")
+        self.ver: str = req.get("ver", None)
-        self.serial: str = req.get("serial", "")
+        self.serial: str = req.get("serial", None)
-        self.ip: str = req.get("ip", "")
+        self.ip: str = req.get("ip", None)
-        self.firm_ver: str = req.get("firm_ver", "")
+        self.firm_ver: str = req.get("firm_ver", None)
-        self.boot_ver: str = req.get("boot_ver", "")
+        self.boot_ver: str = req.get("boot_ver", None)
-        self.encode: str = req.get("encode", "")
+        self.encode: str = req.get("encode", "EUC-JP")
-        self.hops = int(req.get("hops", "0"))
+        self.hops = int(req.get("hops", "-1"))
-        self.format_ver = req.get("format_ver", "2")
+        self.format_ver = float(req.get("format_ver", "1.00"))
-        self.token = int(req.get("token", "0"))
+        self.token: str = req.get("token", "0")
 
-
+class AllnetPowerOnResponse:
-class AllnetPowerOnResponse3:
-    def __init__(self, token) -> None:
-        self.stat = 1
-        self.uri = ""
-        self.host = ""
-        self.place_id = "123"
-        self.name = ""
-        self.nickname = ""
-        self.region0 = "1"
-        self.region_name0 = "W"
-        self.region_name1 = ""
-        self.region_name2 = ""
-        self.region_name3 = ""
-        self.country = "JPN"
-        self.allnet_id = "123"
-        self.client_timezone = "+0900"
-        self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime(
-            "%Y-%m-%dT%H:%M:%SZ"
-        )
-        self.setting = "1"
-        self.res_ver = "3"
-        self.token = str(token)
-
-
-class AllnetPowerOnResponse2:
     def __init__(self) -> None:
         self.stat = 1
         self.uri = ""
         self.host = ""
         self.place_id = "123"
-        self.name = "Test"
+        self.name = "ARTEMiS"
-        self.nickname = "Test123"
+        self.nickname = "ARTEMiS"
         self.region0 = "1"
         self.region_name0 = "W"
-        self.region_name1 = "X"
+        self.region_name1 = ""
-        self.region_name2 = "Y"
+        self.region_name2 = ""
-        self.region_name3 = "Z"
+        self.region_name3 = ""        
-        self.country = "JPN"
+        self.setting = "1"
         self.year = datetime.now().year
         self.month = datetime.now().month
         self.day = datetime.now().day
         self.hour = datetime.now().hour
         self.minute = datetime.now().minute
         self.second = datetime.now().second
-        self.setting = "1"
+
-        self.timezone = "+0900"
+class AllnetPowerOnResponse3(AllnetPowerOnResponse):
+    def __init__(self, token) -> None:
+        super().__init__()
+
+        # Added in v3
+        self.country = "JPN"
+        self.allnet_id = "123"
+        self.client_timezone = "+0900"
+        self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime(
+            "%Y-%m-%dT%H:%M:%SZ"
+        )
+        self.res_ver = "3"
+        self.token = token
+
+        # Removed in v3
+        self.year = None
+        self.month = None
+        self.day = None
+        self.hour = None
+        self.minute = None
+        self.second = None
+
+
+class AllnetPowerOnResponse2(AllnetPowerOnResponse):
+    def __init__(self) -> None:
+        super().__init__()
+
+        # Added in v2
+        self.country = "JPN"
+        self.timezone = "+09:00"
         self.res_class = "PowerOnResponseV2"
 
 
@@ -419,11 +583,119 @@ class AllnetDownloadOrderRequest:
 
 
 class AllnetDownloadOrderResponse:
-    def __init__(self, stat: int = 1, serial: str = "", uri: str = "null") -> None:
+    def __init__(self, stat: int = 1, serial: str = "", uri: str = "") -> None:
         self.stat = stat
         self.serial = serial
         self.uri = uri
 
+class TraceDataType(Enum):
+    CHARGE = 0
+    EVENT = 1
+    CREDIT = 2
+
+class BillingType(Enum):
+    A = 1
+    B = 0
+
+class float5:
+    def __init__(self, n: str = "0") -> None:
+        nf = float(n)
+        if nf > 999.9 or nf < 0:
+            raise ValueError('float5 must be between 0.000 and 999.9 inclusive')
+        
+        return nf
+    
+    @classmethod
+    def to_str(cls, f: float):
+        return f"%.{2 - int(math.log10(f))+1}f" % f
+
+class BillingInfo:
+    def __init__(self, data: Dict) -> None:
+        try:
+            self.keychipid = str(data.get("keychipid", None))
+            self.functype = int(data.get("functype", None))
+            self.gameid = str(data.get("gameid", None))
+            self.gamever = float(data.get("gamever", None))
+            self.boardid = str(data.get("boardid", None))
+            self.tenpoip = str(data.get("tenpoip", None))
+            self.libalibver = float(data.get("libalibver", None))
+            self.datamax = int(data.get("datamax", None))
+            self.billingtype = BillingType(int(data.get("billingtype", None)))
+            self.protocolver = float(data.get("protocolver", None))
+            self.operatingfix = bool(data.get("operatingfix", None))
+            self.traceleft = int(data.get("traceleft", None))
+            self.requestno = int(data.get("requestno", None))
+            self.datesync = bool(data.get("datesync", None))
+            self.timezone = str(data.get("timezone", None))
+            self.date = datetime.strptime(data.get("date", None), BILLING_DT_FORMAT)
+            self.crcerrcnt = int(data.get("crcerrcnt", None))
+            self.memrepair = bool(data.get("memrepair", None))
+            self.playcnt = int(data.get("playcnt", None))
+            self.playlimit = int(data.get("playlimit", None))
+            self.nearfull = int(data.get("nearfull", None))
+        except Exception as e:
+            raise KeyError(e)
+
+class TraceData:
+    def __init__(self, data: Dict) -> None:
+        try:
+            self.crc_err_flg = bool(data.get("cs", None))
+            self.record_number = int(data.get("rn", None))
+            self.seq_number = int(data.get("sn", None))
+            self.trace_type = TraceDataType(int(data.get("tt", None)))
+            self.date_sync_flg = bool(data.get("ds", None))
+            self.date = datetime.strptime(data.get("dt", None), BILLING_DT_FORMAT)
+            self.keychip = str(data.get("kn", None))
+            self.lib_ver = float(data.get("alib", None))
+        except Exception as e:
+            raise KeyError(e)
+
+class TraceDataCharge(TraceData):
+    def __init__(self, data: Dict) -> None:
+        super().__init__(data)
+        try:
+            self.game_id = str(data.get("gi", None))
+            self.game_version = float(data.get("gv", None))
+            self.board_serial = str(data.get("bn", None))
+            self.shop_ip = str(data.get("ti", None))
+            self.play_count = int(data.get("pc", None))
+            self.play_limit = int(data.get("pl", None))
+            self.product_code = int(data.get("ic", None))
+            self.product_count = int(data.get("in", None))
+            self.func_type = int(data.get("kk", None))
+            self.player_number = int(data.get("playerno", None))
+        except Exception as e:
+            raise KeyError(e)
+
+class TraceDataEvent(TraceData):
+    def __init__(self, data: Dict) -> None:
+        super().__init__(data)
+        try:
+            self.message = str(data.get("me", None))
+        except Exception as e:
+            raise KeyError(e)
+
+class TraceDataCredit(TraceData):
+    def __init__(self, data: Dict) -> None:
+        super().__init__(data)
+        try:
+            self.chute_type = int(data.get("cct", None))
+            self.service_type = int(data.get("cst", None))
+            self.operation_type = int(data.get("cop", None))
+            self.coin_rate0 = int(data.get("cr0", None))
+            self.coin_rate1 = int(data.get("cr1", None))
+            self.bonus_addition = int(data.get("cba", None))
+            self.credit_rate = int(data.get("ccr", None))
+            self.credit0 = int(data.get("cc0", None))
+            self.credit1 = int(data.get("cc1", None))
+            self.credit2 = int(data.get("cc2", None))
+            self.credit3 = int(data.get("cc3", None))
+            self.credit4 = int(data.get("cc4", None))
+            self.credit5 = int(data.get("cc5", None))
+            self.credit6 = int(data.get("cc6", None))
+            self.credit7 = int(data.get("cc7", None))
+        except Exception as e:
+            raise KeyError(e)
 
 class BillingResponse:
     def __init__(
@@ -432,20 +704,22 @@ class BillingResponse:
         playlimit_sig: str = "",
         nearfull: str = "",
         nearfull_sig: str = "",
+        request_num: int = 1,
+        protocol_ver: float = 1.000,
         playhistory: str = "000000/0:000000/0:000000/0",
     ) -> None:
-        self.result = "0"
+        self.result = 0
-        self.waitime = "100"
+        self.requestno = request_num
-        self.linelimit = "1"
+        self.traceerase = 1
-        self.message = ""
+        self.fixinterval = 120
+        self.fixlogcnt = 100
         self.playlimit = playlimit
         self.playlimitsig = playlimit_sig
-        self.protocolver = "1.000"
+        self.playhistory = playhistory
         self.nearfull = nearfull
         self.nearfullsig = nearfull_sig
-        self.fixlogincnt = "0"
+        self.linelimit = 100
-        self.fixinterval = "5"
+        self.protocolver = float5.to_str(protocol_ver)
-        self.playhistory = playhistory
         # playhistory -> YYYYMM/C:...
         # YYYY -> 4 digit year, MM -> 2 digit month, C -> Playcount during that period
 
@@ -454,3 +728,86 @@ class AllnetRequestException(Exception):
     def __init__(self, message="") -> None:
         self.message = message
         super().__init__(self.message)
+
+class DLReport:
+    def __init__(self, data: Dict, report_type: DLIMG_TYPE) -> None:
+        self.serial = data.get("serial")
+        self.dfl = data.get("dfl")
+        self.wfl = data.get("wfl")
+        self.tsc = data.get("tsc")
+        self.tdsc = data.get("tdsc")
+        self.at = data.get("at")
+        self.ot = data.get("ot")
+        self.rt = data.get("rt")
+        self.as_ = data.get("as")
+        self.rf_state = data.get("rf_state")
+        self.gd = data.get("gd")
+        self.dav = data.get("dav")
+        self.wdav = data.get("wdav") # app only
+        self.dov = data.get("dov")
+        self.wdov = data.get("wdov") # app only
+        self.__type = report_type
+        self.err = ""
+    
+    def validate(self) -> bool:
+        if  self.serial is None:
+            self.err = "serial not provided"
+            return False
+        
+        if self.dfl is None: 
+            self.err = "dfl not provided"
+            return False
+        
+        if self.wfl is None:
+            self.err = "wfl not provided"
+            return False
+        
+        if self.tsc is None:
+            self.err = "tsc not provided"
+            return False
+        
+        if self.tdsc is None:
+            self.err = "tdsc not provided"
+            return False
+        
+        if self.at is None:
+            self.err = "at not provided"
+            return False
+        
+        if self.ot is None:
+            self.err = "ot not provided"
+            return False
+        
+        if self.rt is None:
+            self.err = "rt not provided"
+            return False
+        
+        if self.as_ is None:
+            self.err = "as not provided"
+            return False
+        
+        if self.rf_state is None:
+            self.err = "rf_state not provided"
+            return False
+        
+        if self.gd is None:
+            self.err = "gd not provided"
+            return False
+        
+        if self.dav is None:
+            self.err = "dav not provided"
+            return False
+        
+        if self.dov is None:
+            self.err = "dov not provided"
+            return False
+        
+        if (self.wdav is None or self.wdov is None) and self.__type == DLIMG_TYPE.app:
+            self.err = "wdav or wdov not provided in app image"
+            return False
+        
+        if (self.wdav is not None or self.wdov is not None) and self.__type == DLIMG_TYPE.opt:
+            self.err = "wdav or wdov provided in opt image"
+            return False
+        
+        return True

core/config.py

@@ -36,12 +36,30 @@ class ServerConfig:
             self.__config, "core", "server", "is_develop", default=True
         )
 
+    @property
+    def threading(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "threading", default=False
+        )
+
     @property
     def log_dir(self) -> str:
         return CoreConfig.get_config_field(
             self.__config, "core", "server", "log_dir", default="logs"
         )
 
+    @property
+    def check_arcade_ip(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "check_arcade_ip", default=False
+        )
+
+    @property
+    def strict_ip_checking(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "strict_ip_checking", default=False
+        )
+
 
 class TitleConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -182,12 +200,24 @@ class AllnetConfig:
             self.__config, "core", "allnet", "port", default=80
         )
 
+    @property
+    def ip_check(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "ip_check", default=False
+        )
+
     @property
     def allow_online_updates(self) -> int:
         return CoreConfig.get_config_field(
             self.__config, "core", "allnet", "allow_online_updates", default=False
         )
 
+    @property
+    def update_cfg_folder(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "update_cfg_folder", default=""
+        )
+
 
 class BillingConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:

core/data/database.py

@@ -1,5 +1,5 @@
 import logging, coloredlogs
-from typing import Optional
+from typing import Optional, Dict, List
 from sqlalchemy.orm import scoped_session, sessionmaker
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy import create_engine
@@ -15,31 +15,48 @@ from core.utils import Utils
 
 
 class Data:
+    current_schema_version = 6
+    engine = None
+    session = None
+    user = None
+    arcade = None
+    card = None
+    base = None
     def __init__(self, cfg: CoreConfig) -> None:
         self.config = cfg
 
         if self.config.database.sha2_password:
             passwd = sha256(self.config.database.password.encode()).digest()
-            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
+            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4"
         else:
-            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
+            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4"
 
-        self.__engine = create_engine(self.__url, pool_recycle=3600)
+        if Data.engine is None:
-        session = sessionmaker(bind=self.__engine, autoflush=True, autocommit=True)
+            Data.engine = create_engine(self.__url, pool_recycle=3600)
-        self.session = scoped_session(session)
+            self.__engine = Data.engine
 
-        self.user = UserData(self.config, self.session)
+        if Data.session is None:
-        self.arcade = ArcadeData(self.config, self.session)
+            s = sessionmaker(bind=Data.engine, autoflush=True, autocommit=True)
-        self.card = CardData(self.config, self.session)
+            Data.session = scoped_session(s)
-        self.base = BaseData(self.config, self.session)
+
-        self.schema_ver_latest = 4
+        if Data.user is None:
+            Data.user = UserData(self.config, self.session)
+        
+        if Data.arcade is None:
+            Data.arcade = ArcadeData(self.config, self.session)
+        
+        if Data.card is None:
+            Data.card = CardData(self.config, self.session)
+        
+        if Data.base is None:
+            Data.base = BaseData(self.config, self.session)
 
-        log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
         self.logger = logging.getLogger("database")
 
         # Prevent the logger from adding handlers multiple times
         if not getattr(self.logger, "handler_set", None):            
+            log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
             fileHandler = TimedRotatingFileHandler(
                 "{0}/{1}.log".format(self.config.server.log_dir, "db"),
                 encoding="utf-8",
@@ -71,11 +88,13 @@ class Data:
         games = Utils.get_all_titles()
         for game_dir, game_mod in games.items():
             try:
-                if hasattr(game_mod, "database") and hasattr(game_mod, "current_schema_version"):
+                if hasattr(game_mod, "database") and hasattr(
+                    game_mod, "current_schema_version"
+                ):
                     game_mod.database(self.config)
                 metadata.create_all(self.__engine.connect())
 
-                self.base.set_schema_ver(
+                self.base.touch_schema_ver(
                     game_mod.current_schema_version, game_mod.game_codes[0]
                 )
 
@@ -84,8 +103,8 @@ class Data:
                     f"Could not load database schema from {game_dir} - {e}"
                 )
 
-        self.logger.info(f"Setting base_schema_ver to {self.schema_ver_latest}")
+        self.logger.info(f"Setting base_schema_ver to {self.current_schema_version}")
-        self.base.set_schema_ver(self.schema_ver_latest)
+        self.base.set_schema_ver(self.current_schema_version)
 
         self.logger.info(
             f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
@@ -129,9 +148,32 @@ class Data:
 
         self.create_database()
 
-    def migrate_database(self, game: str, version: int, action: str) -> None:
+    def migrate_database(self, game: str, version: Optional[int], action: str) -> None:
         old_ver = self.base.get_schema_ver(game)
         sql = ""
+        if version is None:
+            if not game == "CORE":
+                titles = Utils.get_all_titles()
+
+                for folder, mod in titles.items():
+                    if not mod.game_codes[0] == game:
+                        continue
+
+                    if hasattr(mod, "current_schema_version"):
+                        version = mod.current_schema_version
+
+                    else:
+                        self.logger.warning(
+                            f"current_schema_version not found for {folder}"
+                        )
+
+            else:
+                version = self.current_schema_version
+
+        if version is None:
+            self.logger.warning(
+                f"Could not determine latest version for {game}, please specify --version"
+            )
 
         if old_ver is None:
             self.logger.error(
@@ -212,7 +254,7 @@ class Data:
             self.logger.error(f"Failed to create card for owner with id {user_id}")
             return
 
-        self.logger.warn(
+        self.logger.warning(
             f"Successfully created owner with email {email}, access code 00000000000000000000, and password {pw} Make sure to change this password and assign a real card ASAP!"
         )
 
@@ -227,7 +269,7 @@ class Data:
             return
 
         if not should_force:
-            self.logger.warn(
+            self.logger.warning(
                 f"Card already exists for access code {new_ac} (id {new_card['id']}). If you wish to continue, rerun with the '--force' flag."
                 f" All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
             )
@@ -263,17 +305,53 @@ class Data:
                 self.user.delete_user(user["id"])
 
     def autoupgrade(self) -> None:
-        all_games = self.base.get_all_schema_vers()
+        all_game_versions = self.base.get_all_schema_vers()
-        if all_games is None:
+        if all_game_versions is None:
-            self.logger.warn("Failed to get schema versions")
+            self.logger.warning("Failed to get schema versions")
+            return
 
-        for x in all_games:
+        all_games = Utils.get_all_titles()
+        all_games_list: Dict[str, int] = {}
+        for _, mod in all_games.items():
+            if hasattr(mod, "current_schema_version"):
+                all_games_list[mod.game_codes[0]] = mod.current_schema_version
+
+        for x in all_game_versions:
+            failed = False
             game = x["game"].upper()
-            update_ver = 1
+            update_ver = int(x["version"])
-            for y in range(2, 100):
+            latest_ver = all_games_list.get(game, 1)
-                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
+            if game == "CORE":
-                    update_ver = y
+                latest_ver = self.current_schema_version
-                else:
-                    break
 
-            self.migrate_database(game, update_ver, "upgrade")
+            if update_ver == latest_ver:
+                self.logger.info(f"{game} is already latest version")
+                continue
+
+            for y in range(update_ver + 1, latest_ver + 1):
+                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
+                    with open(
+                        f"core/data/schema/versions/{game}_{y}_upgrade.sql",
+                        "r",
+                        encoding="utf-8",
+                    ) as f:
+                        sql = f.read()
+
+                    result = self.base.execute(sql)
+                    if result is None:
+                        self.logger.error(
+                            f"Error execuing sql script for game {game} v{y}!"
+                        )
+                        failed = True
+                        break
+                else:
+                    self.logger.warning(f"Could not find script {game}_{y}_upgrade.sql")
+                    failed = True
+
+            if not failed:
+                self.base.set_schema_ver(latest_ver, game)
+    
+    def show_versions(self) -> None:
+        all_game_versions = self.base.get_all_schema_vers()
+        for ver in all_game_versions:
+            self.logger.info(f"{ver['game']} -> v{ver['version']}")

core/data/schema/arcade.py

@@ -1,9 +1,10 @@
-from typing import Optional, Dict
+from typing import Optional, Dict, List
-from sqlalchemy import Table, Column
+from sqlalchemy import Table, Column, and_, or_
 from sqlalchemy.sql.schema import ForeignKey, PrimaryKeyConstraint
-from sqlalchemy.types import Integer, String, Boolean
+from sqlalchemy.types import Integer, String, Boolean, JSON
 from sqlalchemy.sql import func, select
 from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.engine import Row
 import re
 
 from core.data.schema.base import BaseData, metadata
@@ -21,6 +22,7 @@ arcade = Table(
     Column("city", String(255)),
     Column("region_id", Integer),
     Column("timezone", String(255)),
+    Column("ip", String(39)),
     mysql_charset="utf8mb4",
 )
 
@@ -40,6 +42,9 @@ machine = Table(
     Column("timezone", String(255)),
     Column("ota_enable", Boolean),
     Column("is_cab", Boolean),
+    Column("memo", String(255)),
+    Column("is_cab", Boolean),
+    Column("data", JSON),
     mysql_charset="utf8mb4",
 )
 
@@ -65,7 +70,7 @@ arcade_owner = Table(
 
 
 class ArcadeData(BaseData):
-    def get_machine(self, serial: str = None, id: int = None) -> Optional[Dict]:
+    def get_machine(self, serial: str = None, id: int = None) -> Optional[Row]:
         if serial is not None:
             serial = serial.replace("-", "")
             if len(serial) == 11:
@@ -130,13 +135,20 @@ class ArcadeData(BaseData):
                 f"Failed to update board id for machine {machine_id} -> {boardid}"
             )
 
-    def get_arcade(self, id: int) -> Optional[Dict]:
+    def get_arcade(self, id: int) -> Optional[Row]:
         sql = arcade.select(arcade.c.id == id)
         result = self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
     
+    def get_arcade_machines(self, id: int) -> Optional[List[Row]]:
+        sql = machine.select(machine.c.arcade == id)
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def put_arcade(
         self,
         name: str,
@@ -165,7 +177,21 @@ class ArcadeData(BaseData):
             return None
         return result.lastrowid
 
-    def get_arcade_owners(self, arcade_id: int) -> Optional[Dict]:
+    def get_arcades_managed_by_user(self, user_id: int) -> Optional[List[Row]]:
+        sql = select(arcade).join(arcade_owner, arcade_owner.c.arcade == arcade.c.id).where(arcade_owner.c.user == user_id)
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchall()
+    
+    def get_manager_permissions(self, user_id: int, arcade_id: int) -> Optional[int]:
+        sql = select(arcade_owner.c.permissions).where(and_(arcade_owner.c.user == user_id, arcade_owner.c.arcade == arcade_id))
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchone()
+
+    def get_arcade_owners(self, arcade_id: int) -> Optional[Row]:
         sql = select(arcade_owner).where(arcade_owner.c.arcade == arcade_id)
 
         result = self.execute(sql)
@@ -187,33 +213,14 @@ class ArcadeData(BaseData):
         return f"{platform_code}{platform_rev:02d}A{serial_num:04d}{append:04d}"  # 0x41 = A, 0x52 = R
 
     def validate_keychip_format(self, serial: str) -> bool:
-        serial = serial.replace("-", "")
+        if re.fullmatch(r"^A[0-9]{2}[E|X][-]?[0-9]{2}[A-HJ-NP-Z][0-9]{4}([0-9]{4})?$", serial) is None:
-        if len(serial) != 11 or len(serial) != 15:
-            self.logger.error(
-                f"Serial validate failed: Incorrect length for {serial} (len {len(serial)})"
-            )
-            return False
-
-        platform_code = serial[:4]
-        platform_rev = serial[4:6]
-        const_a = serial[6]
-        num = serial[7:11]
-        append = serial[11:15]
-
-        if re.match("A[7|6]\d[E|X][0|1][0|1|2]A\d{4,8}", serial) is None:
-            self.logger.error(f"Serial validate failed: {serial} failed regex")
-            return False
-
-        if len(append) != 0 or len(append) != 4:
-            self.logger.error(
-                f"Serial validate failed: {serial} had malformed append {append}"
-            )
-            return False
-
-        if len(num) != 4:
-            self.logger.error(
-                f"Serial validate failed: {serial} had malformed number {num}"
-            )
             return False
         
         return True
+
+    def find_arcade_by_name(self, name: str) -> List[Row]:
+        sql = arcade.select(or_(arcade.c.name.like(f"%{name}%"), arcade.c.nickname.like(f"%{name}%")))
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchall()

core/data/schema/base.py

@@ -47,7 +47,7 @@ class BaseData:
         res = None
 
         try:
-            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())} || {opts}")
+            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())}")
             res = self.conn.execute(text(sql), opts)
 
         except SQLAlchemyError as e:
@@ -58,7 +58,7 @@ class BaseData:
             self.logger.error(f"UnicodeEncodeError error {e}")
             return None
 
-        except:
+        except Exception:
             try:
                 res = self.conn.execute(sql, opts)
 
@@ -70,7 +70,7 @@ class BaseData:
                 self.logger.error(f"UnicodeEncodeError error {e}")
                 return None
 
-            except:
+            except Exception:
                 self.logger.error(f"Unknown error")
                 raise
 
@@ -103,6 +103,18 @@ class BaseData:
 
         return row["version"]
     
+    def touch_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
+        sql = insert(schema_ver).values(game=game, version=ver)
+        conflict = sql.on_duplicate_key_update(version=schema_ver.c.version)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(
+                f"Failed to update schema version for game {game} (v{ver})"
+            )
+            return None
+        return result.lastrowid
+
     def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
         sql = insert(schema_ver).values(game=game, version=ver)
         conflict = sql.on_duplicate_key_update(version=ver)

core/data/schema/user.py

@@ -80,6 +80,9 @@ class UserData(BaseData):
         if usr["password"] is None:
             return False
         
+        if passwd is None or not passwd:
+            return False
+
         return bcrypt.checkpw(passwd, usr["password"].encode())
 
     def reset_autoincrement(self, ai_value: int) -> None:
@@ -104,3 +107,17 @@ class UserData(BaseData):
         if result is None:
             return None
         return result.fetchall()
+
+    def find_user_by_email(self, email: str) -> Row:
+        sql = select(aime_user).where(aime_user.c.email == email)
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchone()
+
+    def find_user_by_username(self, username: str) -> List[Row]:
+        sql = aime_user.select(aime_user.c.username.like(f"%{username}%"))
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchall()

core/data/schema/versions/CORE_4_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE machine DROP COLUMN memo;
+ALTER TABLE machine DROP COLUMN is_blacklisted;
+ALTER TABLE machine DROP COLUMN `data`;

core/data/schema/versions/CORE_5_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE arcade DROP COLUMN 'ip';

core/data/schema/versions/CORE_5_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE machine ADD memo varchar(255) NULL;
+ALTER TABLE machine ADD is_blacklisted tinyint(1) NULL;
+ALTER TABLE machine ADD `data` longtext NULL;

core/data/schema/versions/CORE_6_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE arcade ADD ip varchar(39) NULL;

core/data/schema/versions/SBZV_4_rollback.sql

@@ -0,0 +1,9 @@
+ALTER TABLE diva_profile
+    DROP cnp_cid,
+    DROP cnp_val,
+    DROP cnp_rr,
+    DROP cnp_sp,
+    DROP btn_se_eqp,
+    DROP sld_se_eqp,
+    DROP chn_sld_se_eqp,
+    DROP sldr_tch_se_eqp;

core/data/schema/versions/SBZV_5_upgrade.sql

@@ -0,0 +1,9 @@
+ALTER TABLE diva_profile
+    ADD cnp_cid INT NOT NULL DEFAULT -1,
+    ADD cnp_val INT NOT NULL DEFAULT -1,
+    ADD cnp_rr INT NOT NULL DEFAULT -1,
+    ADD cnp_sp VARCHAR(255) NOT NULL DEFAULT "",
+    ADD btn_se_eqp INT NOT NULL DEFAULT -1,
+    ADD sld_se_eqp INT NOT NULL DEFAULT -1,
+    ADD chn_sld_se_eqp INT NOT NULL DEFAULT -1,
+    ADD sldr_tch_se_eqp INT NOT NULL DEFAULT -1;

core/data/schema/versions/SDBT_3_rollback.sql

@@ -0,0 +1,30 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    DROP COLUMN regionId,
+    DROP COLUMN machineType;
+
+ALTER TABLE chuni_static_events
+    DROP COLUMN startDate;
+
+ALTER TABLE chuni_profile_data
+    DROP COLUMN rankUpChallengeResults;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN presetId id INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD PRIMARY KEY(id);
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_uk UNIQUE(id, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY(presetId)
+    REFERENCES chuni_static_login_bonus_preset(id) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_4_upgrade.sql

@@ -0,0 +1,29 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    ADD COLUMN regionId INT,
+    ADD COLUMN machineType INT;
+
+ALTER TABLE chuni_static_events
+    ADD COLUMN startDate TIMESTAMP NOT NULL DEFAULT current_timestamp();
+
+ALTER TABLE chuni_profile_data
+    ADD COLUMN rankUpChallengeResults JSON;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN id presetId INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP INDEX chuni_static_login_bonus_preset_uk;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_pk PRIMARY KEY (presetId, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY (presetId, version)
+    REFERENCES chuni_static_login_bonus_preset(presetId, version) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDDT_4_rollback.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ongeki_static_events
+DROP COLUMN startDate;

core/data/schema/versions/SDDT_5_upgrade.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ongeki_static_events
+ADD COLUMN startDate TIMESTAMP NOT NULL DEFAULT current_timestamp();

core/data/schema/versions/SDEZ_4_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP DEFAULT "2018-01-01 00:00:00.0",
+    CHANGE COLUMN endDate endDate TIMESTAMP DEFAULT "2038-01-01 00:00:00.0";

core/data/schema/versions/SDEZ_5_rollback.sql

@@ -0,0 +1,78 @@
+DELETE FROM mai2_static_event WHERE version < 13;
+UPDATE mai2_static_event SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_music WHERE version < 13;
+UPDATE mai2_static_music SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_ticket WHERE version < 13;
+UPDATE mai2_static_ticket SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_cards WHERE version < 13;
+UPDATE mai2_static_cards SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_detail WHERE version < 13;
+UPDATE mai2_profile_detail SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_extend WHERE version < 13;
+UPDATE mai2_profile_extend SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_option WHERE version < 13;
+UPDATE mai2_profile_option SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_ghost WHERE version < 13;
+UPDATE mai2_profile_ghost SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_rating WHERE version < 13;
+UPDATE mai2_profile_rating SET version = version - 13 WHERE version >= 13;
+
+DROP TABLE maimai_score_best;
+DROP TABLE maimai_playlog;
+DROP TABLE maimai_profile_detail;
+DROP TABLE maimai_profile_option;
+DROP TABLE maimai_profile_web_option;
+DROP TABLE maimai_profile_grade_status;
+
+ALTER TABLE mai2_item_character DROP COLUMN point;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NOT NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NOT NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NOT NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NOT NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NOT NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN `rank` int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NOT NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;

core/data/schema/versions/SDEZ_5_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CHANGE COLUMN endDate endDate TIMESTAMP NOT NULL;

core/data/schema/versions/SDEZ_6_rollback.sql

@@ -0,0 +1 @@
+DROP TABLE aime.mai2_profile_consec_logins;

core/data/schema/versions/SDEZ_6_upgrade.sql

@@ -0,0 +1,62 @@
+UPDATE mai2_static_event SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_music SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_ticket SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_cards SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_detail  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_extend  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_option  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_ghost  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_rating  SET version = version + 13 WHERE version < 1000;
+
+ALTER TABLE mai2_item_character ADD point int(11) NULL;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN `rank` int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;

core/data/schema/versions/SDEZ_7_upgrade.sql

@@ -0,0 +1,9 @@
+CREATE TABLE `mai2_profile_consec_logins` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `user` int(11) NOT NULL,
+  `version` int(11) NOT NULL,
+  `logins` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `mai2_profile_consec_logins_uk` (`user`,`version`),
+  CONSTRAINT `mai2_profile_consec_logins_ibfk_1` FOREIGN KEY (`user`) REFERENCES `aime_user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

core/data/schema/versions/SDFE_3_rollback.sql

@@ -0,0 +1,2 @@
+SET FOREIGN_KEY_CHECKS=0;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDFE_4_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE wacca_profile DROP COLUMN playcount_time_free;

core/data/schema/versions/SDFE_4_upgrade.sql

@@ -0,0 +1 @@
+DELETE FROM wacca_item WHERE type=17 AND item_id=312002;

core/data/schema/versions/SDFE_5_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE wacca_profile ADD playcount_time_free int(11) DEFAULT 0 NULL AFTER playcount_stageup;

core/frontend.py

@@ -9,21 +9,35 @@ from zope.interface import Interface, Attribute, implementer
 from twisted.python.components import registerAdapter
 import jinja2
 import bcrypt
+import re
+from enum import Enum
+from urllib import parse
 
 from core import CoreConfig, Utils
 from core.data import Data
 
 
 class IUserSession(Interface):
-    userId = Attribute("User's ID")
+    user_id = Attribute("User's ID")
+    username = Attribute("User's username")
     current_ip = Attribute("User's current ip address")
     permissions = Attribute("User's permission level")
 
 
+class PermissionOffset(Enum):
+    USER = 0  # Regular user
+    USERMOD = 1  # Can moderate other users
+    ACMOD = 2  # Can add arcades and cabs
+    SYSADMIN = 3  # Can change settings
+    # 4 - 6 reserved for future use
+    OWNER = 7  # Can do anything
+
+
 @implementer(IUserSession)
 class UserSession(object):
     def __init__(self, session):
-        self.userId = 0
+        self.user_id = 0
+        self.username = None
         self.current_ip = "0.0.0.0"
         self.permissions = 0
 
@@ -71,12 +85,18 @@ class FrontendServlet(resource.Resource):
                     game_fe = game_mod.frontend(cfg, self.environment, config_dir)
                     self.game_list.append({"url": game_dir, "name": game_fe.nav_name})
                     fe_game.putChild(game_dir.encode(), game_fe)
-                except:
+
-                    raise
+                except Exception as e:
+                    self.logger.error(
+                        f"Failed to import frontend from {game_dir} because {e}"
+                    )
 
         self.environment.globals["game_list"] = self.game_list
         self.putChild(b"gate", FE_Gate(cfg, self.environment))
         self.putChild(b"user", FE_User(cfg, self.environment))
+        self.putChild(b"sys", FE_System(cfg, self.environment))
+        self.putChild(b"arcade", FE_Arcade(cfg, self.environment))
+        self.putChild(b"cab", FE_Machine(cfg, self.environment))
         self.putChild(b"game", fe_game)
 
         self.logger.info(
@@ -90,6 +110,7 @@ class FrontendServlet(resource.Resource):
             server_name=self.config.server.name,
             title=self.config.server.name,
             game_list=self.game_list,
+            active_page="/",
             sesh=vars(IUserSession(request.getSession())),
         ).encode("utf-16")
 
@@ -118,7 +139,7 @@ class FE_Gate(FE_Base):
 
         sesh = request.getSession()
         usr_sesh = IUserSession(sesh)
-        if usr_sesh.userId > 0:
+        if usr_sesh.user_id > 0:
             return redirectTo(b"/user", request)
 
         if uri.startswith("/gate/create"):
@@ -127,7 +148,7 @@ class FE_Gate(FE_Base):
         if b"e" in request.args:
             try:
                 err = int(request.args[b"e"][0].decode())
-            except:
+            except Exception:
                 err = 0
 
         else:
@@ -137,6 +158,7 @@ class FE_Gate(FE_Base):
         return template.render(
             title=f"{self.core_config.server.name} | Login Gate",
             error=err,
+            active_page="gate",
             sesh=vars(usr_sesh),
         ).encode("utf-16")
 
@@ -145,12 +167,13 @@ class FE_Gate(FE_Base):
         ip = Utils.get_ip_addr(request)
 
         if uri == "/gate/gate.login":
-            access_code: str = request.args[b"access_code"][0].decode()
+            access_code: str = request.args[b"access-code"][0].decode()
             passwd: bytes = request.args[b"passwd"][0]
             if passwd == b"":
                 passwd = None
 
             uid = self.data.card.get_user_id_from_card(access_code)
+            user = self.data.user.get_user(uid)
             if uid is None:
                 return redirectTo(b"/gate?e=1", request)
 
@@ -170,8 +193,10 @@ class FE_Gate(FE_Base):
 
             sesh = request.getSession()
             usr_sesh = IUserSession(sesh)
-            usr_sesh.userId = uid
+            usr_sesh.user_id = uid
+            usr_sesh.username = user["username"]
             usr_sesh.current_ip = ip
+            usr_sesh.permissions = user["permissions"]
 
             return redirectTo(b"/user", request)
 
@@ -179,7 +204,7 @@ class FE_Gate(FE_Base):
             access_code: str = request.args[b"access_code"][0].decode()
             username: str = request.args[b"username"][0]
             email: str = request.args[b"email"][0].decode()
-            passwd: str = request.args[b"passwd"][0]
+            passwd: bytes = request.args[b"passwd"][0]
 
             uid = self.data.card.get_user_id_from_card(access_code)
             if uid is None:
@@ -189,12 +214,12 @@ class FE_Gate(FE_Base):
             hashed = bcrypt.hashpw(passwd, salt)
 
             result = self.data.user.create_user(
-                uid, username, email, hashed.decode(), 1
+                uid, username, email.lower(), hashed.decode(), 1
             )
             if result is None:
                 return redirectTo(b"/gate?e=3", request)
 
-            if not self.data.user.check_password(uid, passwd.encode()):
+            if not self.data.user.check_password(uid, passwd):
                 return redirectTo(b"/gate", request)
 
             return redirectTo(b"/user", request)
@@ -207,26 +232,179 @@ class FE_Gate(FE_Base):
             return redirectTo(b"/gate?e=2", request)
 
         ac = request.args[b"ac"][0].decode()
+        card = self.data.card.get_card_by_access_code(ac)
+        if card is None:
+            return redirectTo(b"/gate?e=1", request)
+
+        user = self.data.user.get_user(card["user"])
+        if user is None:
+            self.logger.warning(
+                f"Card {ac} exists with no/invalid associated user ID {card['user']}"
+            )
+            return redirectTo(b"/gate?e=0", request)
+
+        if user["password"] is not None:
+            return redirectTo(b"/gate?e=1", request)
 
         template = self.environment.get_template("core/frontend/gate/create.jinja")
         return template.render(
             title=f"{self.core_config.server.name} | Create User",
             code=ac,
-            sesh={"userId": 0},
+            active_page="gate",
+            sesh={"user_id": 0, "permissions": 0},
         ).encode("utf-16")
 
 
 class FE_User(FE_Base):
     def render_GET(self, request: Request):
+        uri = request.uri.decode()
         template = self.environment.get_template("core/frontend/user/index.jinja")
 
         sesh: Session = request.getSession()
         usr_sesh = IUserSession(sesh)
-        if usr_sesh.userId == 0:
+        if usr_sesh.user_id == 0:
             return redirectTo(b"/gate", request)
 
+        m = re.match("\/user\/(\d*)", uri)
+
+        if m is not None:
+            usrid = m.group(1)
+            if (
+                usr_sesh.permissions < 1 << PermissionOffset.USERMOD.value
+                or not usrid == usr_sesh.user_id
+            ):
+                return redirectTo(b"/user", request)
+
+        else:
+            usrid = usr_sesh.user_id
+
+        user = self.data.user.get_user(usrid)
+        if user is None:
+            return redirectTo(b"/user", request)
+
+        cards = self.data.card.get_user_cards(usrid)
+        arcades = self.data.arcade.get_arcades_managed_by_user(usrid)
+
+        card_data = []
+        arcade_data = []
+
+        for i, c in enumerate(cards):
+            if c["is_locked"]:
+                status = "Locked"
+            elif c["is_banned"]:
+                status = "Banned"
+            else:
+                status = "Active"
+
+            card_data.append({"index": i+1, "access_code": c["access_code"], "status": status})
+
+        for i, a in enumerate(arcades):
+            arcade_data.append({"index": i+1, "id": a["id"], "name": a["name"]})
+
         return template.render(
-            title=f"{self.core_config.server.name} | Account", sesh=vars(usr_sesh)
+            title=f"{self.core_config.server.name} | Account",
+            sesh=vars(usr_sesh),
+            cards=card_data,
+            arcades=arcade_data,
+            active_page="user",
+        ).encode("utf-16")
+
+    def render_POST(self, request: Request):
+        pass
+
+
+class FE_System(FE_Base):
+    def render_GET(self, request: Request):
+        uri = request.uri.decode()
+        template = self.environment.get_template("core/frontend/sys/index.jinja")
+        usrlist = []
+        aclist = []
+        cablist = []
+
+        sesh: Session = request.getSession()
+        usr_sesh = IUserSession(sesh)
+        if (
+            usr_sesh.user_id == 0
+            or usr_sesh.permissions < 1 << PermissionOffset.USERMOD.value
+        ):
+            return redirectTo(b"/gate", request)
+
+        if uri.startswith("/sys/lookup.user?"):
+            uri_parse = parse.parse_qs(
+                uri.replace("/sys/lookup.user?", "")
+            )  # lop off the first bit
+            uid_search = uri_parse.get("usrId")
+            email_search = uri_parse.get("usrEmail")
+            uname_search = uri_parse.get("usrName")
+
+            if uid_search is not None:
+                u = self.data.user.get_user(uid_search[0])
+                if u is not None:
+                    usrlist.append(u._asdict())
+
+            elif email_search is not None:
+                u = self.data.user.find_user_by_email(email_search[0])
+                if u is not None:
+                    usrlist.append(u._asdict())
+
+            elif uname_search is not None:
+                ul = self.data.user.find_user_by_username(uname_search[0])
+                for u in ul:
+                    usrlist.append(u._asdict())
+
+        elif uri.startswith("/sys/lookup.arcade?"):
+            uri_parse = parse.parse_qs(
+                uri.replace("/sys/lookup.arcade?", "")
+            )  # lop off the first bit
+            ac_id_search = uri_parse.get("arcadeId")
+            ac_name_search = uri_parse.get("arcadeName")
+            ac_user_search = uri_parse.get("arcadeUser")
+
+            if ac_id_search is not None:
+                u = self.data.arcade.get_arcade(ac_id_search[0])
+                if u is not None:
+                    aclist.append(u._asdict())
+
+            elif ac_name_search is not None:
+                ul = self.data.arcade.find_arcade_by_name(ac_name_search[0])
+                for u in ul:
+                    aclist.append(u._asdict())
+
+            elif ac_user_search is not None:
+                ul = self.data.arcade.get_arcades_managed_by_user(ac_user_search[0])
+                for u in ul:
+                    aclist.append(u._asdict())
+
+        elif uri.startswith("/sys/lookup.cab?"):
+            uri_parse = parse.parse_qs(
+                uri.replace("/sys/lookup.cab?", "")
+            )  # lop off the first bit
+            cab_id_search = uri_parse.get("cabId")
+            cab_serial_search = uri_parse.get("cabSerial")
+            cab_acid_search = uri_parse.get("cabAcId")
+
+            if cab_id_search is not None:
+                u = self.data.arcade.get_machine(id=cab_id_search[0])
+                if u is not None:
+                    cablist.append(u._asdict())
+
+            elif cab_serial_search is not None:
+                u = self.data.arcade.get_machine(serial=cab_serial_search[0])
+                if u is not None:
+                    cablist.append(u._asdict())
+
+            elif cab_acid_search is not None:
+                ul = self.data.arcade.get_arcade_machines(cab_acid_search[0])
+                for u in ul:
+                    cablist.append(u._asdict())
+
+        return template.render(
+            title=f"{self.core_config.server.name} | System",
+            sesh=vars(usr_sesh),
+            usrlist=usrlist,
+            aclist=aclist,
+            cablist=cablist,
+            active_page="sys",
         ).encode("utf-16")
 
 
@@ -241,3 +419,56 @@ class FE_Game(FE_Base):
 
     def render_GET(self, request: Request) -> bytes:
         return redirectTo(b"/user", request)
+
+
+class FE_Arcade(FE_Base):
+    def render_GET(self, request: Request):
+        uri = request.uri.decode()
+        template = self.environment.get_template("core/frontend/arcade/index.jinja")
+        managed = []
+
+        sesh: Session = request.getSession()
+        usr_sesh = IUserSession(sesh)
+        if usr_sesh.user_id == 0:
+            return redirectTo(b"/gate", request)
+
+        m = re.match("\/arcade\/(\d*)", uri)
+
+        if m is not None:
+            arcadeid = m.group(1)
+            perms = self.data.arcade.get_manager_permissions(usr_sesh.user_id, arcadeid)
+            arcade = self.data.arcade.get_arcade(arcadeid)
+
+            if perms is None:
+                perms = 0
+
+        else:
+            return redirectTo(b"/user", request)
+
+        return template.render(
+            title=f"{self.core_config.server.name} | Arcade",
+            sesh=vars(usr_sesh),
+            error=0,
+            perms=perms,
+            arcade=arcade._asdict(),
+            active_page="arcade",
+        ).encode("utf-16")
+
+
+class FE_Machine(FE_Base):
+    def render_GET(self, request: Request):
+        uri = request.uri.decode()
+        template = self.environment.get_template("core/frontend/machine/index.jinja")
+
+        sesh: Session = request.getSession()
+        usr_sesh = IUserSession(sesh)
+        if usr_sesh.user_id == 0:
+            return redirectTo(b"/gate", request)
+
+        return template.render(
+            title=f"{self.core_config.server.name} | Machine",
+            sesh=vars(usr_sesh),
+            arcade={},
+            error=0,
+            active_page="machine",
+        ).encode("utf-16")

core/frontend/arcade/index.jinja

@@ -0,0 +1,4 @@
+{% extends "core/frontend/index.jinja" %}
+{% block content %}
+<h1>{{ arcade.name }}</h1>
+{% endblock content %}

core/frontend/gate/create.jinja

@@ -1,24 +1,44 @@
 {% extends "core/frontend/index.jinja" %}
 {% block content %}
-<h1>Create User</h1>
+<div class="row justify-content-md-center form-signin">
-<form id="create" style="max-width: 240px; min-width: 10%;" action="/gate/gate.create" method="post">
+    <div class="col col-lg-4">
-    <div class="form-group row">
+        <form id="create" action="/gate/gate.create" method="post">
-        <label for="access_code">Card Access Code</label><br>
+            <h1 class="h3 mb-3 fw-normal">Sign-up</h1>
-        <input class="form-control" name="access_code" id="access_code" type="text" placeholder="00000000000000000000" value={{ code }} maxlength="20" readonly>
+
+            <div class="form-floating mb-3">
+                <input class="form-control" name="access-code" id="access-code" type="text"
+                    placeholder="00000000000000000000" value={{ code }} maxlength="20" required>
+                <label for="access-code">Access Code*</label>
+            </div>
+            <div class="form-floating mb-3">
+                <input type="text" class="form-control" name="username" id="username" placeholder="username">
+                <label for="username">Username*</label>
+            </div>
+            <div class="form-floating mb-3">
+                <input type="email" class="form-control" name="email" id="email" placeholder="example@example.com">
+                <label for="email">E-Mail*</label>
+            </div>
+            <div class="form-floating mb-3">
+                <input type="password" class="form-control" name="passwd" id="passwd" placeholder="Password">
+                <label for="passwd">Password*</label>
+            </div>
+            
+            <!--
+            <div class="form-check text-start my-3">
+                <input class="form-check-input" type="checkbox" value="remember-me" id="remember-me">
+                <label class="form-check-label" for="remember-me">
+                    Remember me
+                </label>
+            </div>
+            -->
+
+            <div class="alert alert-warning" role="alert">
+                If you have not registered a card with this server, you cannot create a WebUI account!
+            </div>
+
+            <button class="btn btn-primary w-100 py-2" type="submit">Sign-up</button>
+            <!-- <p class="mt-5 mb-3 text-body-secondary">© 2023 ARTEMiS</p>-->
+        </form>
     </div>
-    <div class="form-group row">
+</div>
-        <label for="username">Username</label><br>
-        <input id="username" class="form-control" name="username" type="text" placeholder="username">
-    </div>
-    <div class="form-group row">
-        <label for="email">Email</label><br>
-        <input id="email" class="form-control" name="email" type="email" placeholder="example@example.com">
-    </div>
-    <div class="form-group row">
-        <label for="passwd">Password</label><br>
-        <input id="passwd" class="form-control" name="passwd" type="password" placeholder="password">
-    </div>
-    <p></p>
-    <input id="submit" class="btn btn-primary" style="display: block; margin: 0 auto;" type="submit" value="Create">
-</form>
 {% endblock content %}

core/frontend/gate/gate.jinja

@@ -1,32 +1,52 @@
 {% extends "core/frontend/index.jinja" %}
 {% block content %}
-<h1>Gate</h1>
-{% include "core/frontend/widgets/err_banner.jinja" %}
 <style>
-    /* Chrome, Safari, Edge, Opera */
+    .form-signin input[type="text"] {
-    input::-webkit-outer-spin-button,
+        margin-bottom: -1px;
-    input::-webkit-inner-spin-button {
+        border-bottom-right-radius: 0;
-        -webkit-appearance: none;
+        border-bottom-left-radius: 0;
-        margin: 0;
     }
 
-    /* Firefox */
+    .form-signin input[type="password"] {
-    input[type=number] {
+        margin-bottom: 10px;
-        -moz-appearance: textfield;
+        border-top-left-radius: 0;
+        border-top-right-radius: 0;
     }
 </style>
-<form id="login" style="max-width: 240px; min-width: 10%;" action="/gate/gate.login" method="post">
+<div class="row justify-content-md-center form-signin">
-    <div class="form-group row">
+    <div class="col col-lg-4">
-        <label for="access_code">Card Access Code</label><br>
+        <form id="login" action="/gate/gate.login" method="post">
-        <input form="login" class="form-control" name="access_code" id="access_code" type="number" placeholder="00000000000000000000" maxlength="20" required>
+            <h1 class="h3 mb-3 fw-normal">Login</h1>
+
+            {% include "core/frontend/widgets/err_banner.jinja" %}
+
+            <div class="form-floating">
+                <input form="login" class="form-control" name="access-code" id="access-code" type="text"
+                    placeholder="00000000000000000000" maxlength="20" required>
+                <label for="access-code">Access Code</label>
+            </div>
+            <div class="form-floating">
+                <input form="login" type="password" class="form-control" name="passwd" id="passwd" placeholder="Password">
+                <label for="passwd">Password</label>
+            </div>
+            
+            <!--
+            <div class="form-check text-start my-3">
+                <input class="form-check-input" type="checkbox" value="remember-me" id="remember-me">
+                <label class="form-check-label" for="remember-me">
+                    Remember me
+                </label>
+            </div>
+            -->
+
+            <div class="alert alert-info" role="alert">
+                To register for the WebUI, type in the <strong>access code</strong> of your card, as shown in a game, and leave the
+                password field blank.
+            </div>
+
+            <button class="btn btn-primary w-100 py-2" type="submit">Sign in</button>
+            <!-- <p class="mt-5 mb-3 text-body-secondary">© 2023 ARTEMiS</p>-->
+        </form>
     </div>
-    <div class="form-group row">
+</div>
-        <label for="passwd">Password</label><br>
-        <input id="passwd" class="form-control" name="passwd" type="password" placeholder="password">
-    </div>
-    <p></p>
-    <input id="submit" class="btn btn-primary" style="display: block; margin: 0 auto;" form="login" type="submit" value="Login">
-</form>
-<h6>*To register for the webui, type in the access code of your card, as shown in a game, and leave the password field blank.</h6>
-<h6>*If you have not registered a card with this server, you cannot create a webui account.</h6>
 {% endblock content %}

core/frontend/index.jinja

@@ -1,88 +1,25 @@
 <!DOCTYPE html>
-<html>
+<html lang="en" data-bs-theme="dark">
-    <head>
+
-        <title>{{ title }}</title>
+<head>
-        <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65" crossorigin="anonymous">
+    <meta charset="utf-8">
-        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-kenU1KFdBIe4zVF0s0G1M5b4hcpxyD9F7jL+jjXkk+Q2h455rYXK/7HAuoJl+0I4" crossorigin="anonymous"></script>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
-        <style>
+    <title>{{ title }}</title>
-            html {
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.1/dist/css/bootstrap.min.css" rel="stylesheet"
-                background-color: #181a1b !important;
+        integrity="sha384-4bw+/aepP/YC94hEpVNVgiZdgIC5+VKNBQNGCHeKRQN+PtmoHDEXuppvnDJzQIu9" crossorigin="anonymous">
-                margin: 10px;
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.5/font/bootstrap-icons.css">
-            }
+</head>
-            html {
+
-                color-scheme: dark !important;
+<body>
-            }
-            html, body, input, textarea, select, button, dialog {
-                background-color: #181a1b;
-            }
-            html, body, input, textarea, select, button {
-                border-color: #736b5e;
-                color: #e8e6e3;
-            }
-            a {
-                color: #3391ff;
-            }
-            table {
-                border-color: #545b5e;
-            }
-            ::placeholder {
-                color: #b2aba1;
-            }
-            input:-webkit-autofill,
-            textarea:-webkit-autofill,
-            select:-webkit-autofill {
-                background-color: #404400 !important;
-                color: #e8e6e3 !important;
-            }
-            ::-webkit-scrollbar {
-                background-color: #202324;
-                color: #aba499;
-            }
-            ::-webkit-scrollbar-thumb {
-                background-color: #454a4d;
-            }
-            ::-webkit-scrollbar-thumb:hover {
-                background-color: #575e62;
-            }
-            ::-webkit-scrollbar-thumb:active {
-                background-color: #484e51;
-            }
-            ::-webkit-scrollbar-corner {
-                background-color: #181a1b;
-            }
-            * {
-                scrollbar-color: #454a4d #202324;
-            }
-            ::selection {
-                background-color: #004daa !important;
-                color: #e8e6e3 !important;
-            }
-            ::-moz-selection {
-                background-color: #004daa !important;
-                color: #e8e6e3 !important;
-            }
-            input[type="text"], input[type="text"]:focus, input[type="password"], input[type="password"]:focus, input[type="email"], input[type="email"]:focus {
-                background-color: #202324 !important;
-                color: #e8e6e3;
-            }
-            form {
-                outline: 1px solid grey; 
-                padding: 20px;
-                padding-top: 10px;
-                padding-bottom: 10px;
-            }
-            .err-banner {
-                background-color: #AA0000;
-                padding: 20px;
-                margin-bottom: 10px;
-                width: 15%;
-            }
-        </style>
-    </head>
-    <body>
     {% include "core/frontend/widgets/topbar.jinja" %}
-    {% block content %}
+    <div class="container">
-    <h1>{{ server_name }}</h1>
+        {% block content %}
-    {% endblock content %}
+        <h1>{{ server_name }}</h1>
-    </body>
+        {% endblock content %}
+    </div>
+    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.1/dist/js/bootstrap.bundle.min.js"
+        integrity="sha384-HwwvtgBNo3bZJJLYd8oVXjrBZt8cqVSpeBNS5n7C8IVInixGAoxmnlMuBnhbgrkm"
+        crossorigin="anonymous"></script>
+</body>
+
 </html>

core/frontend/machine/index.jinja

@@ -0,0 +1,5 @@
+{% extends "core/frontend/index.jinja" %}
+{% block content %}
+{% include "core/frontend/widgets/err_banner.jinja" %}
+<h1>Machine Management</h1>
+{% endblock content %}

core/frontend/sys/index.jinja

@@ -0,0 +1,98 @@
+{% extends "core/frontend/index.jinja" %}
+{% block content %}
+<h1>System Management</h1>
+
+<div class="row" id="rowForm">
+    {% if sesh.permissions >= 2 %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="usrLookup" name="usrLookup" action="/sys/lookup.user" class="form-inline">
+            <h3>User Search</h3>
+            <div class="form-group">
+                <label for="usrEmail">Email address</label>
+                <input type="email" class="form-control" id="usrEmail" name="usrEmail" aria-describedby="emailHelp">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="usrName">Username</label>
+                <input type="text" class="form-control" id="usrName" name="usrName">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="usrId">User ID</label>
+                <input type="number" class="form-control" id="usrId" name="usrId">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    {% endif %}
+    {% if sesh.permissions >= 4 %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="arcadeLookup" name="arcadeLookup" action="/sys/lookup.arcade" class="form-inline" >
+            <h3>Arcade Search</h3>
+            <div class="form-group">
+                <label for="arcadeName">Arcade Name</label>
+                <input type="text" class="form-control" id="arcadeName" name="arcadeName">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="arcadeId">Arcade ID</label>
+                <input type="number" class="form-control" id="arcadeId" name="arcadeId">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="arcadeUser">Owner User ID</label>
+                <input type="number" class="form-control" id="arcadeUser" name="arcadeUser">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="cabLookup" name="cabLookup" action="/sys/lookup.cab" class="form-inline" >
+            <h3>Machine Search</h3>
+            <div class="form-group">
+                <label for="cabSerial">Machine Serial</label>
+                <input type="text" class="form-control" id="cabSerial" name="cabSerial">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="cabId">Machine ID</label>
+                <input type="number" class="form-control" id="cabId" name="cabId">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="cabAcId">Arcade ID</label>
+                <input type="number" class="form-control" id="cabAcId" name="cabAcId">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    {% endif %}
+</div>
+<div class="row" id="rowResult" style="margin: 10px;">
+    {% if sesh.permissions >= 2 %}
+    <div id="userSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for usr in usrlist %}
+        <pre><a href=/user/{{ usr.id }}>{{ usr.id }} | {{ usr.username }}</a></pre>
+        {% endfor %}
+    </div>
+    {% endif %}
+    {% if sesh.permissions >= 4 %}
+    <div id="arcadeSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for ac in aclist %}
+        <pre><a href=/arcade/{{ ac.id }}>{{ ac.id }} | {{ ac.name }}</a></pre>
+        {% endfor %}
+    </div
+    ><div id="cabSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for cab in cablist %}
+        <a href=/cab/{{ cab.id }}><pre>{{ cab.id }} | {{ cab.game if cab.game is defined else "ANY "  }} | {{ cab.serial }}</pre></a>
+        {% endfor %}
+    </div>
+    {% endif %}
+</div>
+<div class="row" id="rowAdd">
+
+</div>
+{% endblock content %}

core/frontend/user/index.jinja

@@ -1,4 +1,126 @@
 {% extends "core/frontend/index.jinja" %}
 {% block content %}
-<h1>testing</h1>
+<h1>{{ sesh["username"] }}'s Account</h1>
+
+<div class="card mb-3">
+    <div class="card-body">
+        <h3 class="card-title">Cards</h3>
+        <!--<h4 class="card-subtitle mb-2 text-body-secondary">Card subtitle</h4>-->
+        <p class="card-text">All aime cards are listed here for the given user.</p>
+        <a href="#" data-bs-toggle="modal" data-bs-target="#card-add" class="btn btn-primary mb-3">Add Card</a>
+        {% if cards is defined and cards|length > 0 %}
+        <div class="container">
+            <div class="row">
+                <div class="col-12">
+                    <table class="table table-hover">
+                        <thead>
+                            <tr>
+                                <th scope="col">#</th>
+                                <th scope="col">Access Code</th>
+                                <th scope="col">Status</th>
+                                <th scope="col">Actions</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {% for c in cards %}
+                            <tr class="align-middle">
+                                <th scope="row">{{ c.index }}</th>
+                                <td>{{ c.access_code }}</td>
+                                <td>
+                                    {% if c.status == 'Active'%}
+                                    <span class="badge rounded-pill text-bg-success">Active</span>
+                                    {% elif c.status == 'Locked' %}
+                                    <span class="badge rounded-pill text-bg-warning">Locked</span>
+                                    {% endif %}
+                                </td>
+                                <td>
+                                    {% if c.status == 'Active'%}
+                                    <button type="button" class="btn btn-warning btn-sm">Lock</i></button>
+                                    {% elif c.status == 'Locked' %}
+                                    <button type="button" class="btn btn-success btn-sm">Unlock</i></button>
+                                    {% endif %}
+                                    <button type="button" class="btn btn-danger btn-sm">Delete</i></button>
+                                </td>
+                            </tr>
+                            {% endfor %}
+                        </tbody>
+                    </table>
+                </div>
+            </div>
+        </div>
+        {% else %}
+        <div class="alert alert-warning" role="alert">
+            You have no cards registered to your account. Please add one.
+        </div>
+        {% endif %}
+        <!--<a href="#" data-bs-toggle="modal" data-bs-target="#card-add" class="card-link">Add Card</a>-->
+    </div>
+</div>
+
+<div class="card">
+    <div class="card-body">
+        <h3 class="card-title">Arcades</h3>
+        <!--<h4 class="card-subtitle mb-2 text-body-secondary">Card subtitle</h4>-->
+        <p class="card-text">All arcades connected to the given account are listed here.</p>
+        {% if arcades is defined and arcades|length > 0 %}
+        <div class="container">
+            <div class="row">
+                <div class="col-12">
+                    <table class="table table-hover">
+                        <thead>
+                            <tr>
+                                <th scope="col">#</th>
+                                <th scope="col">Name</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {% for a in arcades %}
+                            <tr class="align-middle clickable-row" data-href=/arcade/{{ a.id }}>
+                                <th scope="row">{{ a.index }}</th>
+                                <td>{{ a.name }}</td>
+                            </tr>
+                            {% endfor %}
+                        </tbody>
+                    </table>
+                </div>
+            </div>
+        </div>
+        {% else %}
+        <div class="alert alert-info" role="alert">
+            You have no arcades connected to your account.
+        </div>
+        {% endif %}
+    </div>
+</div>
+
+<div class="modal fade" id="card-add" tabindex="-1" aria-labelledby="card-add-label" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h1 class="modal-title fs-5" id="card-add-label">Add Card</h1>
+                <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+            </div>
+            <div class="modal-body">
+                <h3 class="fs-5">How to:</h3>
+                Scan your card on any networked game and press the "View Access Code" button (varies by game) and enter
+                the 20
+                digit code below.
+                <hr>
+                FOR AMUSE IC CARDS: DO NOT ENTER THE CODE SHOWN ON THE BACK OF THE CARD ITSELF OR IT WILL NOT WORK!
+                <form>
+                    <div class="form-floating mt-3">
+                        <label for="access-code" class="col-form-label">Access Code</label>
+                        <input maxlength="20" type="text" required pattern="[0-9]{20}" class="form-control"
+                            id="access-code">
+                    </div>
+                </form>
+            </div>
+            <div class="modal-footer">
+                <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+                <button type="button" class="btn btn-primary">Add</button>
+            </div>
+        </div>
+    </div>
+</div>
+
 {% endblock content %}

core/frontend/widgets/err_banner.jinja

@@ -1,12 +1,15 @@
 {% if error > 0 %}
-<div class="err-banner">
+<div class="alert alert-danger" role="alert">
-<h3>Error</h3>
 {% if error == 1 %}
 Card not registered, or wrong password
 {% elif error == 2 %}
 Missing or malformed access code
 {% elif error == 3 %}
 Failed to create user
+{% elif error == 4 %}
+Arcade not found
+{% elif error == 5 %}
+Machine not found
 {% else %}
 An unknown error occoured
 {% endif %}

core/frontend/widgets/topbar.jinja

@@ -1,18 +1,56 @@
-<div style="background: #333; color: #f9f9f9; width: 10%; height: 50px; line-height: 50px; text-align: center; float: left;">
+<nav class="p-3 mb-3 navbar navbar-expand-lg bg-body-tertiary" aria-label="Thirteenth navbar example">
-    Navigation
+    <div class="container">
-</div>
+        <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarsExample11"
-<div style="background: #333; color: #f9f9f9; width: 80%; height: 50px; line-height: 50px; padding-left: 10px; float: left;">
+            aria-controls="navbarsExample11" aria-expanded="false" aria-label="Toggle navigation">
-    <a href=/><button class="btn btn-primary">Home</button></a>&nbsp;
+            <span class="navbar-toggler-icon"></span>
-    {% for game in game_list %}    
+        </button>
-    <a href=game/{{ game.url }}><button class="btn btn-success">{{ game.name }}</button></a>&nbsp;
-    {% endfor %}
-</div>
-</div>
-<div style="background: #333; color: #f9f9f9; width: 10%; height: 50px; line-height: 50px; text-align: center; float: left;">
-    {% if sesh is defined and sesh["userId"] > 0 %}
-    <a href="/user"><button class="btn btn-primary">Account</button></a>
-    {% else %}
-    <a href="/gate"><button class="btn btn-primary">Gate</button></a>
-    {% endif %}
 
-</div>
+        <div class="collapse navbar-collapse d-lg-flex" id="navbarsExample11">
+            <a class="navbar-brand col-lg-3 me-0" href= />ARTEMiS</a>
+            <ul class="navbar-nav col-lg-6 justify-content-lg-center nav-pills">
+                <li class="nav-item">
+                    <a class="nav-link {% if active_page == '/' %}active{% endif %}" aria-current="page" href= />Home</a>
+                </li>
+                {% for game in game_list %}
+                <li class="nav-item">
+                    <a class="nav-link {% if active_page == game.url %}active{% endif %}" href=/game/{{ game.url }}>{{ game.name }}</a>
+                </li>
+                {% endfor %}
+                <!--
+                    <li class="nav-item dropdown">
+                        <a class="nav-link dropdown-toggle" href="#" data-bs-toggle="dropdown"
+                            aria-expanded="false">Dropdown</a>
+                        <ul class="dropdown-menu">
+                            <li><a class="dropdown-item" href="#">Action</a></li>
+                            <li><a class="dropdown-item" href="#">Another action</a></li>
+                            <li><a class="dropdown-item" href="#">Something else here</a></li>
+                        </ul>
+                    </li>
+                    -->
+            </ul>
+            <div class="d-lg-flex col-lg-3 justify-content-lg-end">
+                {% if sesh is defined and sesh["user_id"] > 0 %}
+                <div class="btn-group dropdown">
+                    <a href="#" class="d-block link-body-emphasis text-decoration-none dropdown-toggle"
+                        data-bs-toggle="dropdown" aria-expanded="false">
+                       {{ sesh["username"] }}
+                    </a>
+                    <ul class="dropdown-menu text-small">
+                        {% if sesh["permissions"] >= 2 %}
+                        <li><a class="dropdown-item {% if active_page == 'sys' %}active{% endif %}" href="/sys">System</a></li>
+                        {% endif %}
+                        <li><a class="dropdown-item" href="#">Settings</a></li>
+                        <li><a class="dropdown-item {% if active_page == 'user' %}active{% endif %}" href="/user">Account</a></li>
+                        <li>
+                            <hr class="dropdown-divider">
+                        </li>
+                        <li><a class="dropdown-item" href="#">Sign out</a></li>
+                    </ul>
+                </div>
+                {% else %}
+                <a href="/gate"><button class="btn {% if active_page == 'gate' %}btn-primary{% else %}btn-outline-primary{% endif %}">Login</button></a>
+                {% endif %}
+            </div>
+        </div>
+    </div>
+</nav>

core/mucha.py

@@ -4,6 +4,7 @@ from logging.handlers import TimedRotatingFileHandler
 from twisted.web import resource
 from twisted.web.http import Request
 from datetime import datetime
+from Crypto.Cipher import Blowfish
 import pytz
 
 from core import CoreConfig
@@ -33,8 +34,8 @@ class MuchaServlet:
         self.logger.addHandler(fileHandler)
         self.logger.addHandler(consoleHandler)
 
-        self.logger.setLevel(logging.INFO)
+        self.logger.setLevel(cfg.mucha.loglevel)
-        coloredlogs.install(level=logging.INFO, logger=self.logger, fmt=log_fmt_str)
+        coloredlogs.install(level=cfg.mucha.loglevel, logger=self.logger, fmt=log_fmt_str)
 
         all_titles = Utils.get_all_titles()
 
@@ -46,9 +47,7 @@ class MuchaServlet:
                 if enabled:
                     self.mucha_registry.append(game_cd)
 
-        self.logger.info(
+        self.logger.info(f"Serving {len(self.mucha_registry)} games")
-            f"Serving {len(self.mucha_registry)} games"
-        )
 
     def handle_boardauth(self, request: Request, _: Dict) -> bytes:
         req_dict = self.mucha_preprocess(request.content.getvalue())
@@ -58,19 +57,24 @@ class MuchaServlet:
             self.logger.error(
                 f"Error processing mucha request {request.content.getvalue()}"
             )
-            return b""
+            return b"RESULTS=000"
 
         req = MuchaAuthRequest(req_dict)
+        self.logger.info(f"Boardauth request from {client_ip} for {req.gameVer}")        
         self.logger.debug(f"Mucha request {vars(req)}")
-        self.logger.info(
-            f"Boardauth request from {client_ip} for {req.gameVer}"
-        )
 
         if req.gameCd not in self.mucha_registry:
-            self.logger.warn(f"Unknown gameCd {req.gameCd}")
+            self.logger.warning(f"Unknown gameCd {req.gameCd}")
-            return b""
+            return b"RESULTS=000"
 
         # TODO: Decrypt S/N
+        b_key = b""
+        for x in range(8):
+            b_key += req.sendDate[(x - 1) & 7].encode()
+
+        cipher = Blowfish.new(b_key, Blowfish.MODE_ECB)
+        sn_decrypt = cipher.decrypt(bytes.fromhex(req.serialNum))
+        self.logger.debug(f"Decrypt SN to {sn_decrypt.hex()}")
 
         resp = MuchaAuthResponse(
             f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}"
@@ -88,24 +92,37 @@ class MuchaServlet:
             self.logger.error(
                 f"Error processing mucha request {request.content.getvalue()}"
             )
-            return b""
+            return b"RESULTS=000"
 
         req = MuchaUpdateRequest(req_dict)
+        self.logger.info(f"Updatecheck request from {client_ip} for {req.gameVer}")        
         self.logger.debug(f"Mucha request {vars(req)}")
-        self.logger.info(
-            f"Updatecheck request from {client_ip} for {req.gameVer}"
-        )
 
         if req.gameCd not in self.mucha_registry:
-            self.logger.warn(f"Unknown gameCd {req.gameCd}")
+            self.logger.warning(f"Unknown gameCd {req.gameCd}")
-            return b""
+            return b"RESULTS=000"
 
-        resp = MuchaUpdateResponseStub(req.gameVer)
+        resp = MuchaUpdateResponse(req.gameVer, f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}")
 
         self.logger.debug(f"Mucha response {vars(resp)}")
 
         return self.mucha_postprocess(vars(resp))
 
+    def handle_dlstate(self, request: Request, _: Dict) -> bytes:
+        req_dict = self.mucha_preprocess(request.content.getvalue())
+        client_ip = Utils.get_ip_addr(request)
+
+        if req_dict is None:
+            self.logger.error(
+                f"Error processing mucha request {request.content.getvalue()}"
+            )
+            return b""
+        
+        req = MuchaDownloadStateRequest(req_dict)
+        self.logger.info(f"DownloadState request from {client_ip} for {req.gameCd} -> {req.updateVer}")        
+        self.logger.debug(f"request {vars(req)}")
+        return b"RESULTS=001"
+
     def mucha_preprocess(self, data: bytes) -> Optional[Dict]:
         try:
             ret: Dict[str, Any] = {}
@@ -117,19 +134,17 @@ class MuchaServlet:
 
             return ret
 
-        except:
+        except Exception:
             self.logger.error(f"Error processing mucha request {data}")
             return None
 
     def mucha_postprocess(self, data: dict) -> Optional[bytes]:
         try:
-            urlencode = ""
+            urlencode = "&".join(f"{k}={v}" for k, v in data.items())
-            for k, v in data.items():
-                urlencode += f"{k}={v}&"
 
             return urlencode.encode()
 
-        except:
+        except Exception:
             self.logger.error("Error processing mucha response")
             return None
 
@@ -209,21 +224,120 @@ class MuchaUpdateRequest:
 class MuchaUpdateResponse:
     def __init__(self, game_ver: str, mucha_url: str) -> None:
         self.RESULTS = "001"        
+        self.EXE_VER = game_ver
+
         self.UPDATE_VER_1 = game_ver
-        self.UPDATE_URL_1 = f"https://{mucha_url}/updUrl1/"
+        self.UPDATE_URL_1 = f"http://{mucha_url}/updUrl1/"
-        self.UPDATE_SIZE_1 = "0"
+        self.UPDATE_SIZE_1 = "20"
-        self.UPDATE_CRC_1 = "0000000000000000"
+
-        self.CHECK_URL_1 = f"https://{mucha_url}/checkUrl/"
+        self.CHECK_CRC_1 = "0000000000000000"
-        self.EXE_VER_1 = game_ver
+        self.CHECK_URL_1 = f"http://{mucha_url}/checkUrl/"
+        self.CHECK_SIZE_1 = "20"
+
         self.INFO_SIZE_1 = "0"
         self.COM_SIZE_1 = "0"
         self.COM_TIME_1 = "0"
         self.LAN_INFO_SIZE_1 = "0"
+
         self.USER_ID = ""
         self.PASSWORD = ""
 
+"""
+RESULTS
+EXE_VER
 
+UPDATE_VER_%d
+UPDATE_URL_%d
+UPDATE_SIZE_%d
+
+CHECK_CRC_%d
+CHECK_URL_%d
+CHECK_SIZE_%d
+
+INFO_SIZE_1
+COM_SIZE_1
+COM_TIME_1
+LAN_INFO_SIZE_1
+
+USER_ID
+PASSWORD
+"""
 class MuchaUpdateResponseStub:
     def __init__(self, game_ver: str) -> None:
         self.RESULTS = "001"
         self.UPDATE_VER_1 = game_ver
+
+class MuchaDownloadStateRequest:
+    def __init__(self, request: Dict) -> None:        
+        self.gameCd = request.get("gameCd", "")
+        self.updateVer = request.get("updateVer", "")
+        self.serialNum = request.get("serialNum", "")
+        self.fileSize = request.get("fileSize", "")
+        self.compFileSize = request.get("compFileSize", "")
+        self.boardId = request.get("boardId", "")
+        self.placeId = request.get("placeId", "")
+        self.storeRouterIp = request.get("storeRouterIp", "")
+
+class MuchaDownloadErrorRequest:
+    def __init__(self, request: Dict) -> None:        
+        self.gameCd = request.get("gameCd", "")
+        self.updateVer = request.get("updateVer", "")
+        self.serialNum = request.get("serialNum", "")
+        self.downloadUrl = request.get("downloadUrl", "")
+        self.errCd = request.get("errCd", "")
+        self.errMessage = request.get("errMessage", "")
+        self.boardId = request.get("boardId", "")
+        self.placeId = request.get("placeId", "")
+        self.storeRouterIp = request.get("storeRouterIp", "")
+
+class MuchaRegiAuthRequest:
+    def __init__(self, request: Dict) -> None:        
+        self.gameCd = request.get("gameCd", "")
+        self.serialNum = request.get("serialNum", "") # Encrypted
+        self.countryCd = request.get("countryCd", "")
+        self.registrationCd = request.get("registrationCd", "")
+        self.sendDate = request.get("sendDate", "")
+        self.useToken = request.get("useToken", "")
+        self.allToken = request.get("allToken", "")
+        self.placeId = request.get("placeId", "")
+        self.storeRouterIp = request.get("storeRouterIp", "")
+
+class MuchaRegiAuthResponse:
+    def __init__(self) -> None:
+        self.RESULTS = "001" # 001 = success, 099, 098, 097 = fail, others = fail
+        self.ALL_TOKEN = "0" # Encrypted
+        self.ADD_TOKEN = "0" # Encrypted
+
+class MuchaTokenStateRequest:
+    def __init__(self, request: Dict) -> None:        
+        self.gameCd = request.get("gameCd", "")
+        self.serialNum = request.get("serialNum", "")
+        self.countryCd = request.get("countryCd", "")
+        self.useToken = request.get("useToken", "")
+        self.allToken = request.get("allToken", "")
+        self.placeId = request.get("placeId", "")
+        self.storeRouterIp = request.get("storeRouterIp", "")
+
+class MuchaTokenStateResponse:
+    def __init__(self) -> None:
+        self.RESULTS = "001"
+
+class MuchaTokenMarginStateRequest:
+    def __init__(self, request: Dict) -> None:        
+        self.gameCd = request.get("gameCd", "")
+        self.serialNum = request.get("serialNum", "")
+        self.countryCd = request.get("countryCd", "")
+        self.placeId = request.get("placeId", "")
+        self.limitLowerToken = request.get("limitLowerToken", 0)
+        self.limitUpperToken = request.get("limitUpperToken", 0)
+        self.settlementMonth = request.get("settlementMonth", 0)
+
+class MuchaTokenMarginStateResponse:
+    def __init__(self) -> None:
+        self.RESULTS = "001"
+        self.LIMIT_LOWER_TOKEN = 0
+        self.LIMIT_UPPER_TOKEN = 0
+        self.LAST_SETTLEMENT_MONTH = 0
+        self.LAST_LIMIT_LOWER_TOKEN = 0
+        self.LAST_LIMIT_UPPER_TOKEN = 0
+        self.SETTLEMENT_MONTH = 0

core/title.py

@@ -62,7 +62,7 @@ class TitleServlet:
                             self.title_registry[code] = handler_cls
 
                 else:
-                    self.logger.warn(f"Game {folder} has no get_allnet_info")
+                    self.logger.warning(f"Game {folder} has no get_allnet_info")
 
             else:
                 self.logger.error(f"{folder} missing game_code or index in __init__.py")
@@ -74,28 +74,28 @@ class TitleServlet:
     def render_GET(self, request: Request, endpoints: dict) -> bytes:
         code = endpoints["game"]
         if code not in self.title_registry:
-            self.logger.warn(f"Unknown game code {code}")
+            self.logger.warning(f"Unknown game code {code}")
             request.setResponseCode(404)
             return b""
 
         index = self.title_registry[code]
         if not hasattr(index, "render_GET"):
-            self.logger.warn(f"{code} does not dispatch GET")
+            self.logger.warning(f"{code} does not dispatch GET")
             request.setResponseCode(405)
             return b""
 
-        return index.render_GET(request, endpoints["version"], endpoints["endpoint"])
+        return index.render_GET(request, int(endpoints["version"]), endpoints["endpoint"])
 
     def render_POST(self, request: Request, endpoints: dict) -> bytes:
         code = endpoints["game"]
         if code not in self.title_registry:
-            self.logger.warn(f"Unknown game code {code}")
+            self.logger.warning(f"Unknown game code {code}")
             request.setResponseCode(404)
             return b""
 
         index = self.title_registry[code]
         if not hasattr(index, "render_POST"):
-            self.logger.warn(f"{code} does not dispatch POST")
+            self.logger.warning(f"{code} does not dispatch POST")
             request.setResponseCode(405)
             return b""
 

core/utils.py

@@ -16,7 +16,10 @@ class Utils:
                 if not dir.startswith("__"):
                     try:
                         mod = importlib.import_module(f"titles.{dir}")
-                        ret[dir] = mod
+                        if hasattr(mod, "game_codes") and hasattr(
+                            mod, "index"
+                        ):  # Minimum required to function
+                            ret[dir] = mod
 
                     except ImportError as e:
                         logging.getLogger("core").error(f"get_all_titles: {dir} - {e}")
@@ -25,4 +28,8 @@ class Utils:
 
     @classmethod
     def get_ip_addr(cls, req: Request) -> str:
-        return req.getAllHeaders()[b"x-forwarded-for"].decode() if b"x-forwarded-for" in req.getAllHeaders() else req.getClientAddress().host
+        return (
+            req.getAllHeaders()[b"x-forwarded-for"].decode()
+            if b"x-forwarded-for" in req.getAllHeaders()
+            else req.getClientAddress().host
+        )

dbutils.py

@@ -1,5 +1,6 @@
 import yaml
 import argparse
+import logging
 from core.config import CoreConfig
 from core.data import Data
 from os import path, mkdir, access, W_OK
@@ -32,7 +33,9 @@ if __name__ == "__main__":
 
     cfg = CoreConfig()
     if path.exists(f"{args.config}/core.yaml"):
-        cfg.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
+        cfg_dict = yaml.safe_load(open(f"{args.config}/core.yaml"))
+        cfg_dict.get("database", {})["loglevel"] = "info"
+        cfg.update(cfg_dict)
 
     if not path.exists(cfg.server.log_dir):
         mkdir(cfg.server.log_dir)
@@ -45,7 +48,6 @@ if __name__ == "__main__":
 
     data = Data(cfg)
 
-
     if args.action == "create":
         data.create_database()
 
@@ -54,15 +56,22 @@ if __name__ == "__main__":
 
     elif args.action == "upgrade" or args.action == "rollback":
         if args.version is None:
-            data.logger.error("Must set game and version to migrate to")
+            data.logger.warning("No version set, upgrading to latest")
-            exit(0)
 
         if args.game is None:
-            data.logger.info("No game set, upgrading core schema")
+            data.logger.warning("No game set, upgrading core schema")
-            data.migrate_database("CORE", int(args.version), args.action)
+            data.migrate_database(
+                "CORE",
+                int(args.version) if args.version is not None else None,
+                args.action,
+            )
 
         else:
-            data.migrate_database(args.game, int(args.version), args.action)
+            data.migrate_database(
+                args.game,
+                int(args.version) if args.version is not None else None,
+                args.action,
+            )
 
     elif args.action == "autoupgrade":
         data.autoupgrade()
@@ -76,4 +85,7 @@ if __name__ == "__main__":
     elif args.action == "cleanup":
         data.delete_hanging_users()
     
+    elif args.action == "version":
+        data.show_versions()
+
     data.logger.info("Done")

docker-compose.yml

@@ -0,0 +1,57 @@
+version: "3.9"
+services:
+  app:
+    hostname: ma.app
+    build: .
+    volumes:
+      - ./aime:/app/aime
+
+    environment:
+      CFG_DEV: 1
+      CFG_CORE_SERVER_HOSTNAME: 0.0.0.0
+      CFG_CORE_DATABASE_HOST: ma.db
+      CFG_CORE_MEMCACHED_HOSTNAME: ma.memcached
+      CFG_CORE_AIMEDB_KEY: keyhere
+      CFG_CHUNI_SERVER_LOGLEVEL: debug
+
+    ports:
+    - "80:80"
+    - "8443:8443"
+    - "22345:22345"
+
+    - "8080:8080"
+    - "8090:8090"
+
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    hostname: ma.db
+    image: mysql:8.0.31-debian
+    environment:
+      MYSQL_DATABASE: aime
+      MYSQL_USER: aime
+      MYSQL_PASSWORD: aime
+      MYSQL_ROOT_PASSWORD: AimeRootPassword
+    healthcheck:
+      test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"]
+      timeout: 5s
+      retries: 5
+
+
+  memcached:
+    hostname: ma.memcached
+    image: memcached:1.6.17-bullseye
+
+  phpmyadmin:
+    hostname: ma.phpmyadmin
+    image: phpmyadmin:latest
+    environment:
+      PMA_HOSTS: ma.db
+      PMA_USER: root
+      PMA_PASSWORD: AimeRootPassword
+      APACHE_PORT: 8080
+    ports:
+    - "8080:8080"
+

docs/config.md

@@ -5,6 +5,7 @@
 - `allow_unregistered_serials`: Allows games that do not have registered keychips to connect and authenticate. Disable to restrict who can connect to your server. Recomended to disable for production setups. Default `True`
 - `name`: Name for the server, used by some games in their default MOTDs. Default `ARTEMiS`
 - `is_develop`: Flags that the server is a development instance without a proxy standing in front of it. Setting to `False` tells the server not to listen for SSL, because the proxy should be handling all SSL-related things, among other things. Default `True`
+- `threading`: Flags that `reactor.run` should be called via the `Thread` standard library. May provide a speed boost, but removes the ability to kill the server via `Ctrl + C`. Default: `False`
 - `log_dir`: Directory to store logs. Server MUST have read and write permissions to this directory or you will have issues. Default `logs`
 ## Title
 - `loglevel`: Logging level for the title server. Default `info`

docs/game_specific_info.md

@@ -9,42 +9,44 @@ using the megaime database. Clean installations always create the latest databas
 # Table of content
 
 - [Supported Games](#supported-games)
-    - [Chunithm](#chunithm)
+    - [CHUNITHM](#chunithm)
     - [crossbeats REV.](#crossbeats-rev)
     - [maimai DX](#maimai-dx)
     - [O.N.G.E.K.I.](#o-n-g-e-k-i)
     - [Card Maker](#card-maker)
     - [WACCA](#wacca)
+    - [Sword Art Online Arcade](#sao)
 
 
 # Supported Games
 
 Games listed below have been tested and confirmed working.
 
-## Chunithm
+## CHUNITHM
 
 ### SDBT
 
-| Version ID | Version Name       |
+| Version ID | Version Name          |
-|------------|--------------------|
+|------------|-----------------------|
-| 0          | Chunithm           |
+| 0          | CHUNITHM              |
-| 1          | Chunithm+          |
+| 1          | CHUNITHM PLUS         |
-| 2          | Chunithm Air       |
+| 2          | CHUNITHM AIR          |
-| 3          | Chunithm Air +     |
+| 3          | CHUNITHM AIR PLUS     |
-| 4          | Chunithm Star      |
+| 4          | CHUNITHM STAR         |
-| 5          | Chunithm Star +    |
+| 5          | CHUNITHM STAR PLUS    |
-| 6          | Chunithm Amazon    |
+| 6          | CHUNITHM AMAZON       |
-| 7          | Chunithm Amazon +  |
+| 7          | CHUNITHM AMAZON PLUS  |
-| 8          | Chunithm Crystal   |
+| 8          | CHUNITHM CRYSTAL      |
-| 9          | Chunithm Crystal + |
+| 9          | CHUNITHM CRYSTAL PLUS |
-| 10         | Chunithm Paradise  |
+| 10         | CHUNITHM PARADISE     |
 
 ### SDHD/SDBT
 
-| Version ID | Version Name    |
+| Version ID | Version Name        |
-|------------|-----------------|
+|------------|---------------------|
-| 11         | Chunithm New!!  |
+| 11         | CHUNITHM NEW!!      |
-| 12         | Chunithm New!!+ |
+| 12         | CHUNITHM NEW PLUS!! |
+| 13         | CHUNITHM SUN        |
 
 
 ### Importer
@@ -52,7 +54,7 @@ Games listed below have been tested and confirmed working.
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --series SDBT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --game SDBT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
 ```
 
 The importer for Chunithm will import: Events, Music, Charge Items and Avatar Accesories.
@@ -60,14 +62,33 @@ The importer for Chunithm will import: Events, Music, Charge Items and Avatar Ac
 ### Database upgrade
 
 Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see
-which version is the latest, f.e. `SDBT_3_upgrade.sql`. In order to upgrade to version 3 in this case you need to
+which version is the latest, f.e. `SDBT_4_upgrade.sql`. In order to upgrade to version 4 in this case you need to
 perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDBT --version 2 upgrade
+python dbutils.py --game SDBT upgrade
-python dbutils.py --game SDBT --version 3 upgrade
 ```
 
+### Online Battle
+
+**Only matchmaking (with your imaginary friends) is supported! Online Battle does not (yet?) work!**
+
+The first person to start the Online Battle (now called host) will create a "matching room" with a given `roomId`, after that max 3 other people can join the created room.
+Non used slots during the matchmaking will be filled with CPUs after the timer runs out.
+As soon as a new member will join the room the timer will jump back to 60 secs again.
+Sending those 4 messages to all other users is also working properly.
+In order to use the Online Battle every user needs the same ICF, same rom version and same data version!
+If a room is full a new room will be created if another user starts an Online Battle.
+After a failed Online Battle the room will be deleted. The host is used for the timer countdown, so if the connection failes to the host the timer will stop and could create a "frozen" state.
+
+#### Information/Problems:
+
+- Online Battle uses UDP hole punching and opens port 50201?
+- `reflectorUri` seems related to that?
+- Timer countdown should be handled globally and not by one user
+- Game can freeze or can crash if someone (especially the host) leaves the matchmaking
+
+
 ## crossbeats REV.
 
 ### SDCA
@@ -84,7 +105,7 @@ python dbutils.py --game SDBT --version 3 upgrade
 In order to use the importer you need to use the provided `Export.csv` file:
 
 ```shell
-python read.py --series SDCA --version <version ID> --binfolder titles/cxb/data
+python read.py --game SDCA --version <version ID> --binfolder titles/cxb/data
 ```
 
 The importer for crossbeats REV. will import Music.
@@ -106,36 +127,59 @@ Config file is located in `config/cxb.yaml`.
 
 ### SDEZ
 
-| Version ID | Version Name            |
+| Game Code | Version ID | Version Name            |
-|------------|-------------------------|
+|-----------|------------|-------------------------|
-| 0          | maimai DX               |
+
-| 1          | maimai DX PLUS          |
+
-| 2          | maimai DX Splash        |
+For versions pre-dx
-| 3          | maimai DX Splash PLUS   |
+| Game Code | Version ID | Version Name            |
-| 4          | maimai DX Universe      |
+|-----------|------------|-------------------------|
-| 5          | maimai DX Universe PLUS |
+| SBXL      | 0          | maimai                  |
-| 6          | maimai DX Festival      |
+| SBXL      | 1          | maimai PLUS             |
+| SBZF      | 2          | maimai GreeN            |
+| SBZF      | 3          | maimai GreeN PLUS       |
+| SDBM      | 4          | maimai ORANGE           |
+| SDBM      | 5          | maimai ORANGE PLUS      |
+| SDCQ      | 6          | maimai PiNK             |
+| SDCQ      | 7          | maimai PiNK PLUS        |
+| SDDK      | 8          | maimai MURASAKI         |
+| SDDK      | 9          | maimai MURASAKI PLUS    |
+| SDDZ      | 10         | maimai MILK             |
+| SDDZ      | 11         | maimai MILK PLUS        |
+| SDEY      | 12         | maimai FiNALE           |
+| SDEZ      | 13         | maimai DX               |
+| SDEZ      | 14         | maimai DX PLUS          |
+| SDEZ      | 15         | maimai DX Splash        |
+| SDEZ      | 16         | maimai DX Splash PLUS   |
+| SDEZ      | 17         | maimai DX Universe      |
+| SDEZ      | 18         | maimai DX Universe PLUS |
+| SDEZ      | 19         | maimai DX Festival      |
 
 ### Importer
 
 In order to use the importer locate your game installation folder and execute:
-
+DX:
 ```shell
-python read.py --series SDEZ --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --game <Game Code> --version <Version ID> --binfolder /path/to/StreamingAssets --optfolder /path/to/game/option/folder
+```
+Pre-DX:
+```shell
+python read.py --game <Game Code> --version <Version ID> --binfolder /path/to/data --optfolder /path/to/patch/data
 ```
-
 The importer for maimai DX will import Events, Music and Tickets.
 
-**NOTE: It is required to use the importer because the game will
+The importer for maimai Pre-DX will import Events and Music. Not all games will have patch data. Milk - Finale have file encryption, and need an AES key. That key is not provided by the developers. For games that do use encryption, provide the key, as a hex string, with the `--extra` flag. Ex `--extra 00112233445566778899AABBCCDDEEFF`
-crash without Events!**
+
+**Important: It is required to use the importer because some games may not function properly or even crash without Events!**
 
 ### Database upgrade
 
 Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDEZ_2_upgrade.sql`. In order to upgrade to version 2 in this case you need to perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDEZ --version 2 upgrade
+python dbutils.py --game SDEZ upgrade
 ```
+Pre-Dx uses the same database as DX, so only upgrade using the SDEZ game code!
 
 ## Hatsune Miku Project Diva
 
@@ -152,7 +196,7 @@ python dbutils.py --game SDEZ --version 2 upgrade
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --series SBZV --version <version ID> --binfolder /path/to/game/data/diva --optfolder /path/to/game/data/diva/mdata
+python read.py --game SBZV --version <version ID> --binfolder /path/to/game/data/diva --optfolder /path/to/game/data/diva/mdata
 ```
 
 The importer for Project Diva Arcade will all required data in order to use
@@ -175,9 +219,7 @@ which version is the latest, f.e. `SBZV_4_upgrade.sql`. In order to upgrade to v
 perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SBZV --version 2 upgrade
+python dbutils.py --game SBZV upgrade
-python dbutils.py --game SBZV --version 3 upgrade
-python dbutils.py --game SBZV --version 4 upgrade
 ```
 
 ## O.N.G.E.K.I.
@@ -188,12 +230,12 @@ python dbutils.py --game SBZV --version 4 upgrade
 |------------|----------------------------|
 | 0          | O.N.G.E.K.I.               |
 | 1          | O.N.G.E.K.I. +             |
-| 2          | O.N.G.E.K.I. Summer        |
+| 2          | O.N.G.E.K.I. SUMMER        |
-| 3          | O.N.G.E.K.I. Summer +      |
+| 3          | O.N.G.E.K.I. SUMMER +      |
-| 4          | O.N.G.E.K.I. Red           |
+| 4          | O.N.G.E.K.I. R.E.D.        |
-| 5          | O.N.G.E.K.I. Red +         |
+| 5          | O.N.G.E.K.I. R.E.D. +      |
-| 6          | O.N.G.E.K.I. Bright        |
+| 6          | O.N.G.E.K.I. bright        |
-| 7          | O.N.G.E.K.I. Bright Memory |
+| 7          | O.N.G.E.K.I. bright MEMORY |
 
 
 ### Importer
@@ -201,7 +243,7 @@ python dbutils.py --game SBZV --version 4 upgrade
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --series SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --game SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
 ```
 
 The importer for O.N.G.E.K.I. will all all Cards, Music and Events.
@@ -225,9 +267,7 @@ which version is the latest, f.e. `SDDT_4_upgrade.sql`. In order to upgrade to v
 perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDDT --version 2 upgrade
+python dbutils.py --game SDDT upgrade
-python dbutils.py --game SDDT --version 3 upgrade
-python dbutils.py --game SDDT --version 4 upgrade
 ```
 
 ## Card Maker
@@ -236,21 +276,21 @@ python dbutils.py --game SDDT --version 4 upgrade
 
 | Version ID | Version Name    |
 |------------|-----------------|
-| 0          | Card Maker 1.34 |
+| 0          | Card Maker 1.30 |
 | 1          | Card Maker 1.35 |
 
 
 ### Support status
 
-* Card Maker 1.34:
+* Card Maker 1.30:
-  * Chunithm New!!: Yes
+  * CHUNITHM NEW!!: Yes
-  * maimai DX Universe: Yes
+  * maimai DX UNiVERSE: Yes
-  * O.N.G.E.K.I. Bright: Yes
+  * O.N.G.E.K.I. bright: Yes
 
 * Card Maker 1.35:
-  * Chunithm New!!+: Yes
+  * CHUNITHM SUN: Yes (NEW PLUS!! up to A032)
-  * maimai DX Universe PLUS: Yes
+  * maimai DX FESTiVAL: Yes (up to A035) (UNiVERSE PLUS up to A031)
-  * O.N.G.E.K.I. Bright Memory: Yes
+  * O.N.G.E.K.I. bright MEMORY: Yes
 
 
 ### Importer
@@ -259,28 +299,63 @@ In order to use the importer you need to use the provided `.csv` files (which ar
 option folders:
 
 ```shell
-python read.py --series SDED --version <version ID> --binfolder titles/cm/cm_data --optfolder /path/to/cardmaker/option/folder
+python read.py --game SDED --version <version ID> --binfolder titles/cm/cm_data --optfolder /path/to/cardmaker/option/folder
 ```
 
 **If you haven't already executed the O.N.G.E.K.I. importer, make sure you import all cards!**
 
 ```shell
-python read.py --series SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --game SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
 ```
 
-Also make sure to import all maimai and Chunithm data as well:
+Also make sure to import all maimai DX and CHUNITHM data as well:
 
 ```shell
-python read.py --series SDED --version <version ID> --binfolder /path/to/cardmaker/CardMaker_Data
+python read.py --game SDED --version <version ID> --binfolder /path/to/cardmaker/CardMaker_Data
 ```
 
-The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai/Chunithm) and the hardcoded
+The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai DX/CHUNITHM) and the hardcoded
 Cards for each Gacha (O.N.G.E.K.I. only).
 
 **NOTE: Without executing the importer Card Maker WILL NOT work!**
 
 
-### O.N.G.E.K.I. Gachas
+### Config setup
+
+Make sure to update your `config/cardmaker.yaml` with the correct version for each game. To get the current version required to run a specific game, open every opt (Axxx) folder descending until you find all three folders:
+
+- `MU3`: O.N.G.E.K.I.
+- `MAI`: maimai DX
+- `CHU`: CHUNITHM
+
+Inside each folder is a `DataConfig.xml` file, for example:
+
+`MU3/DataConfig.xml`:
+```xml
+  <cardMakerVersion>
+    <major>1</major>
+    <minor>35</minor>
+    <release>3</release>
+  </cardMakerVersion>
+```
+
+Now update your `config/cardmaker.yaml` with the correct version number, for example:
+
+```yaml
+version:
+  1: # Card Maker 1.35
+    ongeki: 1.35.03
+```
+
+For now you also need to update your `config/ongeki.yaml` with the correct version number, for example:
+
+```yaml
+version:
+  7: # O.N.G.E.K.I. bright MEMORY
+    card_maker: 1.35.03
+```
+
+### O.N.G.E.K.I.
 
 Gacha "無料ガチャ" can only pull from the free cards with the following probabilities: 94%: R, 5% SR and 1% chance of
 getting an SSR card
@@ -293,20 +368,24 @@ and 3% chance of getting an SSR card
 All other (limited) gachas can pull from every card added to ongeki_static_cards but with the promoted cards
 (click on the green button under the banner) having a 10 times higher chance to get pulled
 
-### Chunithm Gachas
+### CHUNITHM
 
-All cards in Chunithm (basically just the characters) have the same rarity to it just pulls randomly from all cards
+All cards in CHUNITHM (basically just the characters) have the same rarity to it just pulls randomly from all cards
 from a given gacha but made sure you cannot pull the same card twice in the same 5 times gacha roll.
 
+### maimai DX
+
+Printed maimai DX cards: Freedom (`cardTypeId=6`) or Gold Pass (`cardTypeId=4`) can now be selected during the login process. You can only have ONE Freedom and ONE Gold Pass active at a given time. The cards will expire after 15 days.
+
+Thanks GetzeAvenue for the `selectedCardList` rarity hint!
+
 ### Notes
 
-Card Maker 1.34 will only load an O.N.G.E.K.I. Bright profile (1.30). Card Maker 1.35 will only load an O.N.G.E.K.I.
+Card Maker 1.30-1.34 will only load an O.N.G.E.K.I. Bright profile (1.30). Card Maker 1.35+ will only load an O.N.G.E.K.I.
 Bright Memory profile (1.35).
-The gachas inside the `ongeki.yaml` will make sure only the right gacha ids for the right CM version will be loaded.
+The gachas inside the `config/ongeki.yaml` will make sure only the right gacha ids for the right CM version will be loaded.
 Gacha IDs up to 1140 will be loaded for CM 1.34 and all gachas will be loaded for CM 1.35.
 
-**NOTE: There is currently no way to load/use the (printed) maimai DX cards!**
-
 ## WACCA
 
 ### SDFE
@@ -325,7 +404,7 @@ Gacha IDs up to 1140 will be loaded for CM 1.34 and all gachas will be loaded fo
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --series SDFE --version <version ID> --binfolder /path/to/game/WindowsNoEditor/Mercury/Content
+python read.py --game SDFE --version <version ID> --binfolder /path/to/game/WindowsNoEditor/Mercury/Content
 ```
 
 The importer for WACCA will import all Music data.
@@ -347,6 +426,91 @@ Config file is located in `config/wacca.yaml`.
 Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDFE_3_upgrade.sql`. In order to upgrade to version 3 in this case you need to perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDFE --version 2 upgrade
+python dbutils.py --game SDFE upgrade
-python dbutils.py --game SDFE --version 3 upgrade
 ```
+
+### VIP Rewards
+Below is a list of VIP rewards. Currently, VIP is not implemented, and thus these are not obtainable. These 23 rewards were distributed once per month for VIP users on the real network.
+
+	Plates:
+		211004 リッチ
+		211018 特盛えりざべす
+		211025 イースター
+		211026 特盛りりぃ
+		311004 ファンシー
+		311005 インカンテーション
+		311014 夜明け
+		311015 ネイビー
+		311016 特盛るーん
+	
+	Ring Colors:
+		203002 Gold Rushイエロー
+		203009 トロピカル
+		303005 ネイチャー
+	
+	Icons:
+		202020 どらみんぐ
+		202063 ユニコーン
+		202086 ゴリラ
+		302014 ローズ
+		302015 ファラオ
+		302045 肉球
+		302046 WACCA
+		302047 WACCA Lily
+		302048 WACCA Reverse
+	
+	Note Sound Effect:
+		205002 テニス
+		205008 シャワー
+		305003 タンバリンMk-Ⅱ
+
+## SAO
+
+### SDEW
+
+| Version ID | Version Name  |
+|------------|---------------|
+| 0          | SAO           |
+
+
+### Importer
+
+In order to use the importer locate your game installation folder and execute:
+
+```shell
+python read.py --game SDEW --version <version ID> --binfolder /path/to/game/extractedassets
+```
+
+The importer for SAO will import all items, heroes, support skills and titles data.
+
+### Config
+
+Config file is located in `config/sao.yaml`.
+
+| Option             | Info                                                                        |
+|--------------------|-----------------------------------------------------------------------------|
+| `hostname`         | Changes the server listening address for Mucha                              |
+| `port`             | Changes the listing port                                                    |
+| `auto_register`    | Allows the game to handle the automatic registration of new cards           |
+
+
+### Database upgrade
+
+Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDEW_1_upgrade.sql`. In order to upgrade to version 3 in this case you need to perform all previous updates as well:
+
+```shell
+python dbutils.py --game SDEW upgrade
+```
+
+### Notes
+- Defrag Match will crash at loading
+- Co-Op Online is not supported
+- Shop is not functionnal
+- Player title is currently static and cannot be changed in-game
+- QR Card Scanning currently only load a static hero
+
+### Credits for SAO support:
+
+- Midorica - Limited Network Support
+- Dniel97 - Helping with network base
+- tungnotpunk - Source

entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [[ -z "${CFG_DEV}" ]]; then
+    echo Production mode
+    python3 index.py
+else
+    echo Development mode
+    python3 dbutils.py create
+    nodemon -w aime --legacy-watch index.py
+fi
+

example_config/cardmaker.yaml

@@ -1,3 +1,13 @@
 server:
   enable: True
   loglevel: "info"
+
+version:
+  0:
+    ongeki: 1.30.01
+    chuni: 2.00.00
+    maimai: 1.20.00
+  1:
+    ongeki: 1.35.03
+    chuni: 2.10.00
+    maimai: 1.30.00

example_config/chuni.yaml

@@ -15,6 +15,9 @@ version:
   12:
     rom: 2.05.00
     data: 2.05.00
+  13:
+    rom: 2.10.00
+    data: 2.10.00
 
 crypto:
   encrypted_only: False

example_config/core.yaml

@@ -4,7 +4,10 @@ server:
   allow_unregistered_serials: True
   name: "ARTEMiS"
   is_develop: True
+  threading: False
   log_dir: "logs"
+  check_arcade_ip: False
+  strict_ip_checking: False
 
 title:
   loglevel: "info"
@@ -31,7 +34,9 @@ frontend:
 allnet:
   loglevel: "info"
   port: 80
+  ip_check: False
   allow_online_updates: False
+  update_cfg_folder: ""
 
 billing:
   port: 8443

example_config/idz.yaml

@@ -0,0 +1,11 @@
+server:
+  enable: True
+  loglevel: "info"
+  hostname: ""
+  news: ""
+  aes_key: ""
+
+ports:
+  userdb: 10000
+  match: 10010
+  echo: 10020

example_config/mai2.yaml

@@ -1,3 +1,14 @@
 server:
   enable: True
   loglevel: "info"
+
+deliver:
+  enable: False
+  udbdl_enable: False
+  content_folder: ""
+
+uploads:
+  photos: False
+  photos_dir: ""
+  movies: False
+  movies_dir: ""

example_config/ongeki.yaml

@@ -29,3 +29,9 @@ gachas:
     - 1156
     - 1163
     - 1164
+
+version:
+  6:
+    card_maker: 1.30.01
+  7:
+    card_maker: 1.35.03

example_config/pokken.yaml

@@ -2,8 +2,11 @@ server:
   hostname: "localhost"
   enable: True
   loglevel: "info"
-  port: 9000
-  port_stun: 9001
-  port_turn: 9002
-  port_admission: 9003
   auto_register: True
+  enable_matching: False
+  stun_server_host: "stunserver.stunprotocol.org"
+  stun_server_port: 3478
+
+ports:
+  game: 9000
+  admission: 9001

example_config/sao.yaml

@@ -0,0 +1,6 @@
+server:
+  hostname: "localhost"
+  enable: True
+  loglevel: "info"
+  port: 9000
+  auto_register: True

example_config/wacca.yaml

@@ -29,5 +29,8 @@ gates:
     - 17
     - 18
     - 19
+    - 20
     - 21
     - 22
+    - 23
+    - 24

index.py

@@ -11,7 +11,7 @@ from twisted.web import server, resource
 from twisted.internet import reactor, endpoints
 from twisted.web.http import Request
 from routes import Mapper
-
+from threading import Thread
 
 class HttpDispatcher(resource.Resource):
     def __init__(self, cfg: CoreConfig, config_dir: str):
@@ -26,7 +26,23 @@ class HttpDispatcher(resource.Resource):
         self.title = TitleServlet(cfg, config_dir)
         self.mucha = MuchaServlet(cfg, config_dir)
 
+        self.map_get.connect(
+            "allnet_downloadorder_ini",
+            "/dl/ini/{file}",
+            controller="allnet",
+            action="handle_dlorder_ini",
+            conditions=dict(method=["GET"]),
+        )
+
         self.map_post.connect(
+            "allnet_downloadorder_report",
+            "/report-api/Report",
+            controller="allnet",
+            action="handle_dlorder_report",
+            conditions=dict(method=["POST"]),
+        )
+
+        self.map_get.connect(
             "allnet_ping",
             "/naomitest.html",
             controller="allnet",
@@ -47,6 +63,27 @@ class HttpDispatcher(resource.Resource):
             action="handle_dlorder",
             conditions=dict(method=["POST"]),
         )
+        self.map_post.connect(
+            "allnet_loaderstaterecorder",
+            "/sys/servlet/LoaderStateRecorder",
+            controller="allnet",
+            action="handle_loaderstaterecorder",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "allnet_alive",
+            "/sys/servlet/Alive",
+            controller="allnet",
+            action="handle_alive",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_get.connect(
+            "allnet_alive",
+            "/sys/servlet/Alive",
+            controller="allnet",
+            action="handle_alive",
+            conditions=dict(method=["GET"]),
+        )
         self.map_post.connect(
             "allnet_billing",
             "/request",
@@ -62,6 +99,7 @@ class HttpDispatcher(resource.Resource):
             conditions=dict(method=["POST"]),
         )
 
+        # Maintain compatability
         self.map_post.connect(
             "mucha_boardauth",
             "/mucha/boardauth.do",
@@ -76,6 +114,35 @@ class HttpDispatcher(resource.Resource):
             action="handle_updatecheck",
             conditions=dict(method=["POST"]),
         )
+        self.map_post.connect(
+            "mucha_dlstate",
+            "/mucha/downloadstate.do",
+            controller="mucha",
+            action="handle_dlstate",
+            conditions=dict(method=["POST"]),
+        )
+
+        self.map_post.connect(
+            "mucha_boardauth",
+            "/mucha_front/boardauth.do",
+            controller="mucha",
+            action="handle_boardauth",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "mucha_updatacheck",
+            "/mucha_front/updatacheck.do",
+            controller="mucha",
+            action="handle_updatecheck",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "mucha_dlstate",
+            "/mucha_front/downloadstate.do",
+            controller="mucha",
+            action="handle_dlstate",
+            conditions=dict(method=["POST"]),
+        )
 
         self.map_get.connect(
             "title_get",
@@ -95,7 +162,6 @@ class HttpDispatcher(resource.Resource):
         )
 
     def render_GET(self, request: Request) -> bytes:
-        self.logger.debug(request.uri)
         test = self.map_get.match(request.uri.decode())
         client_ip = Utils.get_ip_addr(request)
 
@@ -145,9 +211,16 @@ class HttpDispatcher(resource.Resource):
 
         if type(ret) == str:
             return ret.encode()
-        elif type(ret) == bytes:
+        
+        elif type(ret) == bytes or type(ret) == tuple: # allow for bytes or tuple (data, response code) responses
             return ret
+        
+        elif ret is None:
+            self.logger.warning(f"None returned by controller for {request.uri.decode()} endpoint")
+            return b""
+        
         else:
+            self.logger.warning(f"Unknown data type returned by controller for {request.uri.decode()} endpoint")
             return b""
 
 
@@ -240,4 +313,7 @@ if __name__ == "__main__":
             server.Site(dispatcher)
         )
 
-    reactor.run()  # type: ignore
+    if cfg.server.threading:
+        Thread(target=reactor.run, args=(False,)).start()
+    else:
+        reactor.run()

read.py

@@ -43,11 +43,11 @@ class BaseReader:
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Import Game Information")
     parser.add_argument(
-        "--series",
+        "--game",
         action="store",
         type=str,
         required=True,
-        help="The game series we are importing.",
+        help="The game we are importing.",
     )
     parser.add_argument(
         "--version",
@@ -109,7 +109,7 @@ if __name__ == "__main__":
     logger.setLevel(log_lv)
     coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
 
-    if args.series is None or args.version is None:
+    if args.game is None or args.version is None:
         logger.error("Game or version not specified")
         parser.print_help()
         exit(1)
@@ -134,9 +134,8 @@ if __name__ == "__main__":
     titles = Utils.get_all_titles()
 
     for dir, mod in titles.items():
-        if args.series in mod.game_codes:
+        if args.game in mod.game_codes:
-            handler = mod.reader(config, args.version,
+            handler = mod.reader(config, args.version, bin_arg, opt_arg, args.extra)
-                                 bin_arg, opt_arg, args.extra)
             handler.read()
 
     logger.info("Done")

readme.md

@@ -2,30 +2,36 @@
 A network service emulator for games running SEGA'S ALL.NET service, and similar.
 
 # Supported games
-Games listed below have been tested and confirmed working. Only game versions older then the current one in active use in arcades (n-0) or current game versions older then a year (y-1) are supported.
+Games listed below have been tested and confirmed working. Only game versions older then the version currently active in arcades, or games versions that have not recieved a major update in over one year, are supported.
-+ Chunithm
-    + All versions up to New!! Plus
 
-+ Crossbeats Rev
++ CHUNITHM
+    + All versions up to SUN
+
++ crossbeats REV.
     + All versions + omnimix
 
 + maimai DX
-    + All versions up to Festival
+    + All versions up to FESTiVAL
 
-+ Hatsune Miku Arcade
++ Hatsune Miku: Project DIVA Arcade
     + All versions
 
 + Card Maker
-    + 1.34.xx
+    + 1.30
-    + 1.35.xx
+    + 1.35
 
-+ Ongeki
++ O.N.G.E.K.I.
-    + All versions up to Bright Memory
+    + All versions up to bright MEMORY
 
-+ Wacca
++ WACCA
     + Lily R
     + Reverse
 
++ POKKÉN TOURNAMENT
+    + Final Online
+
++ Sword Art Online Arcade (partial support)
+    + Final
 
 ## Requirements
 - python 3 (tested working with 3.9 and 3.10, other versions YMMV)

requirements.txt

@@ -16,3 +16,5 @@ Routes
 bcrypt
 jinja2
 protobuf
+autobahn
+pillow

titles/chuni/__init__.py

@@ -7,4 +7,4 @@ index = ChuniServlet
 database = ChuniData
 reader = ChuniReader
 game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW]
-current_schema_version = 3
+current_schema_version = 4

titles/chuni/base.py

@@ -4,7 +4,7 @@ from datetime import datetime, timedelta
 from time import strftime
 
 import pytz
-from typing import Dict, Any
+from typing import Dict, Any, List
 
 from core.config import CoreConfig
 from titles.chuni.const import ChuniConstants
@@ -44,13 +44,15 @@ class ChuniBase:
             # check if a user already has some pogress and if not add the
             # login bonus entry
             user_login_bonus = self.data.item.get_login_bonus(
-                user_id, self.version, preset["id"]
+                user_id, self.version, preset["presetId"]
             )
             if user_login_bonus is None:
-                self.data.item.put_login_bonus(user_id, self.version, preset["id"])
+                self.data.item.put_login_bonus(
+                    user_id, self.version, preset["presetId"]
+                )
                 # yeah i'm lazy
                 user_login_bonus = self.data.item.get_login_bonus(
-                    user_id, self.version, preset["id"]
+                    user_id, self.version, preset["presetId"]
                 )
 
             # skip the login bonus entirely if its already finished
@@ -66,13 +68,13 @@ class ChuniBase:
                 last_update_date = datetime.now()
 
                 all_login_boni = self.data.static.get_login_bonus(
-                    self.version, preset["id"]
+                    self.version, preset["presetId"]
                 )
 
                 # skip the current bonus preset if no boni were found
                 if all_login_boni is None or len(all_login_boni) < 1:
-                    self.logger.warn(
+                    self.logger.warning(
-                        f"No bonus entries found for bonus preset {preset['id']}"
+                        f"No bonus entries found for bonus preset {preset['presetId']}"
                     )
                     continue
 
@@ -83,14 +85,14 @@ class ChuniBase:
                 if bonus_count > max_needed_days:
                     # assume that all login preset ids under 3000 needs to be
                     # looped, like 30 and 40 are looped, 40 does not work?
-                    if preset["id"] < 3000:
+                    if preset["presetId"] < 3000:
                         bonus_count = 1
                     else:
                         is_finished = True
 
                 # grab the item for the corresponding day
                 login_item = self.data.static.get_login_bonus_by_required_days(
-                    self.version, preset["id"], bonus_count
+                    self.version, preset["presetId"], bonus_count
                 )
                 if login_item is not None:
                     # now add the present to the database so the
@@ -108,7 +110,7 @@ class ChuniBase:
                 self.data.item.put_login_bonus(
                     user_id,
                     self.version,
-                    preset["id"],
+                    preset["presetId"],
                     bonusCount=bonus_count,
                     lastUpdateDate=last_update_date,
                     isWatched=False,
@@ -147,7 +149,7 @@ class ChuniBase:
         game_events = self.data.static.get_enabled_events(self.version)
 
         if game_events is None or len(game_events) == 0:
-            self.logger.warn("No enabled events, did you run the reader?")
+            self.logger.warning("No enabled events, did you run the reader?")
             return {
                 "type": data["type"],
                 "length": 0,
@@ -156,12 +158,18 @@ class ChuniBase:
 
         event_list = []
         for evt_row in game_events:
-            tmp = {}
+            event_list.append(
-            tmp["id"] = evt_row["eventId"]
+                {
-            tmp["type"] = evt_row["type"]
+                    "id": evt_row["eventId"],
-            tmp["startDate"] = "2017-12-05 07:00:00.0"
+                    "type": evt_row["type"],
-            tmp["endDate"] = "2099-12-31 00:00:00.0"
+                    # actually use the startDate from the import so it
-            event_list.append(tmp)
+                    # properly shows all the events when new ones are imported
+                    "startDate": datetime.strftime(
+                        evt_row["startDate"], "%Y-%m-%d %H:%M:%S"
+                    ),
+                    "endDate": "2099-12-31 00:00:00",
+                }
+            )
 
         return {
             "type": data["type"],
@@ -228,29 +236,36 @@ class ChuniBase:
     def handle_get_user_character_api_request(self, data: Dict) -> Dict:
         characters = self.data.item.get_characters(data["userId"])
         if characters is None:
-            return {}
+            return {
-        next_idx = -1
+                "userId": data["userId"],
+                "length": 0,
+                "nextIndex": -1,
+                "userCharacterList": [],
+            }
 
-        characterList = []
+        character_list = []
-        for x in range(int(data["nextIndex"]), len(characters)):
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(characters)):
             tmp = characters[x]._asdict()
             tmp.pop("user")
             tmp.pop("id")
-            characterList.append(tmp)
+            character_list.append(tmp)
 
-            if len(characterList) >= int(data["maxCount"]):
+            if len(character_list) >= max_ct:
                 break
 
-        if len(characterList) >= int(data["maxCount"]) and len(characters) > int(
+        if len(characters) >= next_idx + max_ct:
-            data["maxCount"]
+            next_idx += max_ct
-        ) + int(data["nextIndex"]):
+        else:
-            next_idx = int(data["maxCount"]) + int(data["nextIndex"]) + 1
+            next_idx = -1
 
         return {
             "userId": data["userId"],
-            "length": len(characterList),
+            "length": len(character_list),
             "nextIndex": next_idx,
-            "userCharacterList": characterList,
+            "userCharacterList": character_list,
         }
 
     def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
@@ -292,8 +307,8 @@ class ChuniBase:
             if len(user_course_list) >= max_ct:
                 break
 
-        if len(user_course_list) >= max_ct:
+        if len(user_course_list) >= next_idx + max_ct:
-            next_idx = next_idx + max_ct
+            next_idx += max_ct
         else:
             next_idx = -1
 
@@ -347,12 +362,23 @@ class ChuniBase:
         }
 
     def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
+        user_fav_item_list = []
+
+        # still needs to be implemented on WebUI
+        # 1: Music, 3: Character
+        fav_list = self.data.item.get_all_favorites(
+            data["userId"], self.version, fav_kind=int(data["kind"])
+        )
+        if fav_list is not None:
+            for fav in fav_list:
+                user_fav_item_list.append({"id": fav["favId"]})
+
         return {
             "userId": data["userId"],
-            "length": 0,
+            "length": len(user_fav_item_list),
             "kind": data["kind"],
             "nextIndex": -1,
-            "userFavoriteItemList": [],
+            "userFavoriteItemList": user_fav_item_list,
         }
 
     def handle_get_user_favorite_music_api_request(self, data: Dict) -> Dict:
@@ -375,7 +401,7 @@ class ChuniBase:
                 "userItemList": [],
             }
 
-        items: list[Dict[str, Any]] = []
+        items: List[Dict[str, Any]] = []
         for i in range(next_idx, len(user_item_list)):
             tmp = user_item_list[i]._asdict()
             tmp.pop("user")
@@ -387,13 +413,13 @@ class ChuniBase:
         xout = kind * 10000000000 + next_idx + len(items)
 
         if len(items) < int(data["maxCount"]):
-            nextIndex = 0
+            next_idx = 0
         else:
-            nextIndex = xout
+            next_idx = xout
 
         return {
             "userId": data["userId"],
-            "nextIndex": nextIndex,
+            "nextIndex": next_idx,
             "itemKind": kind,
             "length": len(items),
             "userItemList": items,
@@ -452,6 +478,7 @@ class ChuniBase:
                 "nextIndex": -1,
                 "userMusicList": [],  # 240
             }
+
         song_list = []
         next_idx = int(data["nextIndex"])
         max_ct = int(data["maxCount"])
@@ -474,10 +501,10 @@ class ChuniBase:
             if len(song_list) >= max_ct:
                 break
 
-        if len(song_list) >= max_ct:
+        if len(song_list) >= next_idx + max_ct:
             next_idx += max_ct
         else:
-            next_idx = 0
+            next_idx = -1
 
         return {
             "userId": data["userId"],
@@ -617,18 +644,21 @@ class ChuniBase:
                 upsert["userData"][0]["userName"] = self.read_wtf8(
                     upsert["userData"][0]["userName"]
                 )
-            except:
+            except Exception:
                 pass
 
             self.data.profile.put_profile_data(
                 user_id, self.version, upsert["userData"][0]
             )
+
         if "userDataEx" in upsert:
             self.data.profile.put_profile_data_ex(
                 user_id, self.version, upsert["userDataEx"][0]
             )
+
         if "userGameOption" in upsert:
             self.data.profile.put_profile_option(user_id, upsert["userGameOption"][0])
+
         if "userGameOptionEx" in upsert:
             self.data.profile.put_profile_option_ex(
                 user_id, upsert["userGameOptionEx"][0]
@@ -672,6 +702,10 @@ class ChuniBase:
 
         if "userPlaylogList" in upsert:
             for playlog in upsert["userPlaylogList"]:
+                # convert the player names to utf-8
+                playlog["playedUserName1"] = self.read_wtf8(playlog["playedUserName1"])
+                playlog["playedUserName2"] = self.read_wtf8(playlog["playedUserName2"])
+                playlog["playedUserName3"] = self.read_wtf8(playlog["playedUserName3"])
                 self.data.score.put_playlog(user_id, playlog)
 
         if "userTeamPoint" in upsert:

titles/chuni/const.py

@@ -17,21 +17,23 @@ class ChuniConstants:
     VER_CHUNITHM_PARADISE = 10
     VER_CHUNITHM_NEW = 11
     VER_CHUNITHM_NEW_PLUS = 12
+    VER_CHUNITHM_SUN = 13
 
     VERSION_NAMES = [
-        "Chunithm",
+        "CHUNITHM",
-        "Chunithm+",
+        "CHUNITHM PLUS",
-        "Chunithm Air",
+        "CHUNITHM AIR",
-        "Chunithm Air+",
+        "CHUNITHM AIR PLUS",
-        "Chunithm Star",
+        "CHUNITHM STAR",
-        "Chunithm Star+",
+        "CHUNITHM STAR  PLUS",
-        "Chunithm Amazon",
+        "CHUNITHM AMAZON",
-        "Chunithm Amazon+",
+        "CHUNITHM AMAZON PLUS",
-        "Chunithm Crystal",
+        "CHUNITHM CRYSTAL",
-        "Chunithm Crystal+",
+        "CHUNITHM CRYSTAL PLUS",
-        "Chunithm Paradise",
+        "CHUNITHM PARADISE",
-        "Chunithm New!!",
+        "CHUNITHM NEW!!",
-        "Chunithm New!!+",
+        "CHUNITHM NEW PLUS!!",
+        "CHUNITHM SUN"
     ]
 
     @classmethod

titles/chuni/index.py

@@ -29,6 +29,7 @@ from titles.chuni.crystalplus import ChuniCrystalPlus
 from titles.chuni.paradise import ChuniParadise
 from titles.chuni.new import ChuniNew
 from titles.chuni.newplus import ChuniNewPlus
+from titles.chuni.sun import ChuniSun
 
 
 class ChuniServlet:
@@ -55,6 +56,7 @@ class ChuniServlet:
             ChuniParadise,
             ChuniNew,
             ChuniNewPlus,
+            ChuniSun,
         ]
 
         self.logger = logging.getLogger("chuni")
@@ -89,14 +91,29 @@ class ChuniServlet:
 
             self.hash_table[version] = {}
 
-            method_list = [method for method in dir(self.versions[version]) if not method.startswith('__')]
+            method_list = [
+                method
+                for method in dir(self.versions[version])
+                if not method.startswith("__")
+            ]
             for method in method_list:
                 method_fixed = inflection.camelize(method)[6:-7]
-                hash = PBKDF2(method_fixed, bytes.fromhex(keys[2]), 128, count=44, hmac_hash_module=SHA1)
+                # number of iterations was changed to 70 in SUN
+                iter_count = 70 if version >= ChuniConstants.VER_CHUNITHM_SUN else 44
+                hash = PBKDF2(
+                    method_fixed,
+                    bytes.fromhex(keys[2]),
+                    128,
+                    count=iter_count,
+                    hmac_hash_module=SHA1,
+                )
 
-                self.hash_table[version][hash.hex()] = method_fixed
+                hashed_name = hash.hex()[:32] # truncate unused bytes like the game does
+                self.hash_table[version][hashed_name] = method_fixed
 
-                self.logger.debug(f"Hashed v{version} method {method_fixed} with {bytes.fromhex(keys[2])} to get {hash.hex()}")
+                self.logger.debug(
+                    f"Hashed v{version} method {method_fixed} with {bytes.fromhex(keys[2])} to get {hash.hex()}"
+                )
 
     @classmethod
     def get_allnet_info(
@@ -133,30 +150,32 @@ class ChuniServlet:
 
         if version < 105:  # 1.0
             internal_ver = ChuniConstants.VER_CHUNITHM
-        elif version >= 105 and version < 110:  # Plus
+        elif version >= 105 and version < 110:  # PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_PLUS
-        elif version >= 110 and version < 115:  # Air
+        elif version >= 110 and version < 115:  # AIR
             internal_ver = ChuniConstants.VER_CHUNITHM_AIR
-        elif version >= 115 and version < 120:  # Air Plus
+        elif version >= 115 and version < 120:  # AIR PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_AIR_PLUS
-        elif version >= 120 and version < 125:  # Star
+        elif version >= 120 and version < 125:  # STAR
             internal_ver = ChuniConstants.VER_CHUNITHM_STAR
-        elif version >= 125 and version < 130:  # Star Plus
+        elif version >= 125 and version < 130:  # STAR PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_STAR_PLUS
-        elif version >= 130 and version < 135:  # Amazon
+        elif version >= 130 and version < 135:  # AMAZON
             internal_ver = ChuniConstants.VER_CHUNITHM_AMAZON
-        elif version >= 135 and version < 140:  # Amazon Plus
+        elif version >= 135 and version < 140:  # AMAZON PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_AMAZON_PLUS
-        elif version >= 140 and version < 145:  # Crystal
+        elif version >= 140 and version < 145:  # CRYSTAL
             internal_ver = ChuniConstants.VER_CHUNITHM_CRYSTAL
-        elif version >= 145 and version < 150:  # Crystal Plus
+        elif version >= 145 and version < 150:  # CRYSTAL PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS
-        elif version >= 150 and version < 200:  # Paradise
+        elif version >= 150 and version < 200:  # PARADISE
             internal_ver = ChuniConstants.VER_CHUNITHM_PARADISE
-        elif version >= 200 and version < 205:  # New
+        elif version >= 200 and version < 205:  # NEW!!
             internal_ver = ChuniConstants.VER_CHUNITHM_NEW
-        elif version >= 205 and version < 210:  # New Plus
+        elif version >= 205 and version < 210:  # NEW PLUS!!
             internal_ver = ChuniConstants.VER_CHUNITHM_NEW_PLUS
+        elif version >= 210:  # SUN
+            internal_ver = ChuniConstants.VER_CHUNITHM_SUN
 
         if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:
             # If we get a 32 character long hex string, it's a hash and we're
@@ -167,11 +186,15 @@ class ChuniServlet:
 
             else:
                 if internal_ver not in self.hash_table:
-                    self.logger.error(f"v{version} does not support encryption or no keys entered")
+                    self.logger.error(
+                        f"v{version} does not support encryption or no keys entered"
+                    )
                     return zlib.compress(b'{"stat": "0"}')
 
                 elif endpoint.lower() not in self.hash_table[internal_ver]:
-                    self.logger.error(f"No hash found for v{version} endpoint {endpoint}")
+                    self.logger.error(
+                        f"No hash found for v{version} endpoint {endpoint}"
+                    )
                     return zlib.compress(b'{"stat": "0"}')
 
                 endpoint = self.hash_table[internal_ver][endpoint.lower()]
@@ -193,7 +216,11 @@ class ChuniServlet:
 
             encrtped = True
 
-        if not encrtped and self.game_cfg.crypto.encrypted_only and internal_ver >= ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS:
+        if (
+            not encrtped
+            and self.game_cfg.crypto.encrypted_only
+            and internal_ver >= ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS
+        ):
             self.logger.error(
                 f"Unencrypted v{version} {endpoint} request, but config is set to encrypted only: {req_raw}"
             )
@@ -210,9 +237,7 @@ class ChuniServlet:
 
         req_data = json.loads(unzip)
 
-        self.logger.info(
+        self.logger.info(f"v{version} {endpoint} request from {client_ip}")
-            f"v{version} {endpoint} request from {client_ip}"
-        )
         self.logger.debug(req_data)
 
         func_to_find = "handle_" + inflection.underscore(endpoint) + "_request"

titles/chuni/new.py

@@ -23,41 +23,44 @@ class ChuniNew(ChuniBase):
         self.version = ChuniConstants.VER_CHUNITHM_NEW
 
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        # use UTC time and convert it to JST time by adding +9
+        # matching therefore starts one hour before and lasts for 8 hours
         match_start = datetime.strftime(
-            datetime.now() - timedelta(hours=10), self.date_time_format
+            datetime.utcnow() + timedelta(hours=8), self.date_time_format
         )
         match_end = datetime.strftime(
-            datetime.now() + timedelta(hours=10), self.date_time_format
+            datetime.utcnow() + timedelta(hours=16), self.date_time_format
         )
         reboot_start = datetime.strftime(
-            datetime.now() - timedelta(hours=11), self.date_time_format
+            datetime.utcnow() + timedelta(hours=6), self.date_time_format
         )
         reboot_end = datetime.strftime(
-            datetime.now() - timedelta(hours=10), self.date_time_format
+            datetime.utcnow() + timedelta(hours=7), self.date_time_format
         )
         return {
             "gameSetting": {
-                "isMaintenance": "false",
+                "isMaintenance": False,
                 "requestInterval": 10,
                 "rebootStartTime": reboot_start,
                 "rebootEndTime": reboot_end,
-                "isBackgroundDistribute": "false",
+                "isBackgroundDistribute": False,
                 "maxCountCharacter": 300,
                 "maxCountItem": 300,
                 "maxCountMusic": 300,
                 "matchStartTime": match_start,
                 "matchEndTime": match_end,
-                "matchTimeLimit": 99,
+                "matchTimeLimit": 60,
                 "matchErrorLimit": 9999,
                 "romVersion": self.game_cfg.version.version(self.version)["rom"],
                 "dataVersion": self.game_cfg.version.version(self.version)["data"],
                 "matchingUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
                 "matchingUriX": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
+                # might be really important for online battle to connect the cabs via UDP port 50201
                 "udpHolePunchUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
                 "reflectorUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
             },
-            "isDumpUpload": "false",
+            "isDumpUpload": False,
-            "isAou": "false",
+            "isAou": False,
         }
 
     def handle_remove_token_api_request(self, data: Dict) -> Dict:
@@ -468,3 +471,162 @@ class ChuniNew(ChuniBase):
             self.data.item.put_user_print_state(user_id, id=order_id, hasCompleted=True)
 
         return {"returnCode": "1", "apiName": "CMUpsertUserPrintCancelApi"}
+
+    def handle_ping_request(self, data: Dict) -> Dict:
+        # matchmaking ping request
+        return {"returnCode": "1"}
+
+    def handle_begin_matching_api_request(self, data: Dict) -> Dict:
+        room_id = 1
+        # check if there is a free matching room
+        matching_room = self.data.item.get_oldest_free_matching(self.version)
+
+        if matching_room is None:
+            # grab the latest roomId and add 1 for the new room
+            newest_matching = self.data.item.get_newest_matching(self.version)
+            if newest_matching is not None:
+                room_id = newest_matching["roomId"] + 1
+
+            # fix userName WTF8
+            new_member = data["matchingMemberInfo"]
+            new_member["userName"] = self.read_wtf8(new_member["userName"])
+
+            # create the new room with room_id and the current user id (host)
+            # user id is required for the countdown later on
+            self.data.item.put_matching(
+                self.version, room_id, [new_member], user_id=new_member["userId"]
+            )
+
+            # get the newly created matching room
+            matching_room = self.data.item.get_matching(self.version, room_id)
+        else:
+            # a room already exists, so just add the new member to it
+            matching_member_list = matching_room["matchingMemberInfoList"]
+            # fix userName WTF8
+            new_member = data["matchingMemberInfo"]
+            new_member["userName"] = self.read_wtf8(new_member["userName"])
+            matching_member_list.append(new_member)
+
+            # add the updated room to the database, make sure to set isFull correctly!
+            self.data.item.put_matching(
+                self.version,
+                matching_room["roomId"],
+                matching_member_list,
+                user_id=matching_room["user"],
+                is_full=True if len(matching_member_list) >= 4 else False,
+            )
+
+        matching_wait = {
+            "isFinish": False,
+            "restMSec": matching_room["restMSec"],  # in sec
+            "pollingInterval": 1,  # in sec
+            "matchingMemberInfoList": matching_room["matchingMemberInfoList"],
+        }
+
+        return {"roomId": 1, "matchingWaitState": matching_wait}
+
+    def handle_end_matching_api_request(self, data: Dict) -> Dict:
+        matching_room = self.data.item.get_matching(self.version, data["roomId"])
+        members = matching_room["matchingMemberInfoList"]
+
+        # only set the host user to role 1 every other to 0?
+        role_list = [
+            {"role": 1} if m["userId"] == matching_room["user"] else {"role": 0}
+            for m in members
+        ]
+
+        self.data.item.put_matching(
+            self.version,
+            matching_room["roomId"],
+            members,
+            user_id=matching_room["user"],
+            rest_sec=0,  # make sure to always set 0
+            is_full=True,  # and full, so no one can join
+        )
+
+        return {
+            "matchingResult": 1,  # needs to be 1 for successful matching
+            "matchingMemberInfoList": members,
+            # no idea, maybe to differentiate between CPUs and real players?
+            "matchingMemberRoleList": role_list,
+            # TCP/UDP connection?
+            "reflectorUri": f"{self.core_cfg.title.hostname}",
+        }
+
+    def handle_remove_matching_member_api_request(self, data: Dict) -> Dict:
+        # get all matching rooms, because Chuni only returns the userId
+        # not the actual roomId
+        matching_rooms = self.data.item.get_all_matchings(self.version)
+        if matching_rooms is None:
+            return {"returnCode": "1"}
+
+        for room in matching_rooms:
+            old_members = room["matchingMemberInfoList"]
+            new_members = [m for m in old_members if m["userId"] != data["userId"]]
+
+            # if nothing changed go to the next room
+            if len(old_members) == len(new_members):
+                continue
+
+            # if the last user got removed, delete the matching room
+            if len(new_members) <= 0:
+                self.data.item.delete_matching(self.version, room["roomId"])
+            else:
+                # remove the user from the room
+                self.data.item.put_matching(
+                    self.version,
+                    room["roomId"],
+                    new_members,
+                    user_id=room["user"],
+                    rest_sec=room["restMSec"],
+                )
+
+        return {"returnCode": "1"}
+
+    def handle_get_matching_state_api_request(self, data: Dict) -> Dict:
+        polling_interval = 1
+        # get the current active room
+        matching_room = self.data.item.get_matching(self.version, data["roomId"])
+        members = matching_room["matchingMemberInfoList"]
+        rest_sec = matching_room["restMSec"]
+
+        # grab the current member
+        current_member = data["matchingMemberInfo"]
+
+        # only the host user can decrease the countdown
+        if matching_room["user"] == int(current_member["userId"]):
+            # cap the restMSec to 0
+            if rest_sec > 0:
+                rest_sec -= polling_interval
+            else:
+                rest_sec = 0
+
+        # update the members in order to recieve messages
+        for i, member in enumerate(members):
+            if member["userId"] == current_member["userId"]:
+                # replace the old user data with the current user data,
+                # also parse WTF-8 everytime
+                current_member["userName"] = self.read_wtf8(current_member["userName"])
+                members[i] = current_member
+
+        self.data.item.put_matching(
+            self.version,
+            data["roomId"],
+            members,
+            rest_sec=rest_sec,
+            user_id=matching_room["user"],
+        )
+
+        # only add the other members to the list
+        diff_members = [m for m in members if m["userId"] != current_member["userId"]]
+
+        matching_wait = {
+            # makes no difference? Always use False?
+            "isFinish": True if rest_sec == 0 else False,
+            "restMSec": rest_sec,
+            "pollingInterval": polling_interval,
+            # the current user needs to be the first one?
+            "matchingMemberInfoList": [current_member] + diff_members,
+        }
+
+        return {"matchingWaitState": matching_wait}

titles/chuni/newplus.py

@@ -13,8 +13,12 @@ class ChuniNewPlus(ChuniNew):
 
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
         ret = super().handle_get_game_setting_api_request(data)
-        ret["gameSetting"]["romVersion"] = self.game_cfg.version.version(self.version)["rom"]
+        ret["gameSetting"]["romVersion"] = self.game_cfg.version.version(self.version)[
-        ret["gameSetting"]["dataVersion"] = self.game_cfg.version.version(self.version)["data"]
+            "rom"
+        ]
+        ret["gameSetting"]["dataVersion"] = self.game_cfg.version.version(self.version)[
+            "data"
+        ]
         ret["gameSetting"][
             "matchingUri"
         ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/205/ChuniServlet/"
@@ -32,6 +36,6 @@ class ChuniNewPlus(ChuniNew):
     def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
         user_data = super().handle_cm_get_user_preview_api_request(data)
 
-        # hardcode lastDataVersion for CardMaker 1.35
+        # hardcode lastDataVersion for CardMaker 1.35 A028
         user_data["lastDataVersion"] = "2.05.00"
         return user_data

titles/chuni/read.py

@@ -67,7 +67,7 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted login bonus preset {id}")
                     else:
-                        self.logger.warn(f"Failed to insert login bonus preset {id}")
+                        self.logger.warning(f"Failed to insert login bonus preset {id}")
 
                     for bonus in xml_root.find("infos").findall("LoginBonusDataInfo"):
                         for name in bonus.findall("loginBonusName"):
@@ -113,7 +113,7 @@ class ChuniReader(BaseReader):
                                 if result is not None:
                                     self.logger.info(f"Inserted login bonus {bonus_id}")
                                 else:
-                                    self.logger.warn(
+                                    self.logger.warning(
                                         f"Failed to insert login bonus {bonus_id}"
                                     )
 
@@ -138,7 +138,7 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted event {id}")
                     else:
-                        self.logger.warn(f"Failed to insert event {id}")
+                        self.logger.warning(f"Failed to insert event {id}")
 
     def read_music(self, music_dir: str) -> None:
         for root, dirs, files in walk(music_dir):
@@ -200,7 +200,7 @@ class ChuniReader(BaseReader):
                                         f"Inserted music {song_id} chart {chart_id}"
                                     )
                                 else:
-                                    self.logger.warn(
+                                    self.logger.warning(
                                         f"Failed to insert music {song_id} chart {chart_id}"
                                     )
 
@@ -232,7 +232,7 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted charge {id}")
                     else:
-                        self.logger.warn(f"Failed to insert charge {id}")
+                        self.logger.warning(f"Failed to insert charge {id}")
 
     def read_avatar(self, avatar_dir: str) -> None:
         for root, dirs, files in walk(avatar_dir):
@@ -259,4 +259,4 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted avatarAccessory {id}")
                     else:
-                        self.logger.warn(f"Failed to insert avatarAccessory {id}")
+                        self.logger.warning(f"Failed to insert avatarAccessory {id}")

titles/chuni/schema/item.py

@@ -1,5 +1,12 @@
 from typing import Dict, List, Optional
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
+from sqlalchemy import (
+    Table,
+    Column,
+    UniqueConstraint,
+    PrimaryKeyConstraint,
+    and_,
+    delete,
+)
 from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.schema import ForeignKey
@@ -203,8 +210,141 @@ login_bonus = Table(
     mysql_charset="utf8mb4",
 )
 
+favorite = Table(
+    "chuni_item_favorite",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("favId", Integer, nullable=False),
+    Column("favKind", Integer, nullable=False, server_default="1"),
+    UniqueConstraint("version", "user", "favId", name="chuni_item_favorite_uk"),
+    mysql_charset="utf8mb4",
+)
+
+matching = Table(
+    "chuni_item_matching",
+    metadata,
+    Column("roomId", Integer, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("restMSec", Integer, nullable=False, server_default="60"),
+    Column("isFull", Boolean, nullable=False, server_default="0"),
+    PrimaryKeyConstraint("roomId", "version", name="chuni_item_matching_pk"),
+    Column("matchingMemberInfoList", JSON, nullable=False),
+    mysql_charset="utf8mb4",
+)
+
 
 class ChuniItemData(BaseData):
+    def get_oldest_free_matching(self, version: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version,
+                matching.c.isFull == False
+            )
+        ).order_by(matching.c.roomId.asc())
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_newest_matching(self, version: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version
+            )
+        ).order_by(matching.c.roomId.desc())
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_all_matchings(self, version: int) -> Optional[List[Row]]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_matching(self, version: int, room_id: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(matching.c.version == version, matching.c.roomId == room_id)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_matching(
+        self,
+        version: int,
+        room_id: int,
+        matching_member_info_list: List,
+        user_id: int = None,
+        rest_sec: int = 60,
+        is_full: bool = False
+    ) -> Optional[int]:
+        sql = insert(matching).values(
+            roomId=room_id,
+            version=version,
+            restMSec=rest_sec,
+            user=user_id,
+            isFull=is_full,
+            matchingMemberInfoList=matching_member_info_list,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            restMSec=rest_sec, matchingMemberInfoList=matching_member_info_list
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def delete_matching(self, version: int, room_id: int):
+        sql = delete(matching).where(
+            and_(matching.c.roomId == room_id, matching.c.version == version)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def get_all_favorites(
+        self, user_id: int, version: int, fav_kind: int = 1
+    ) -> Optional[List[Row]]:
+        sql = favorite.select(
+            and_(
+                favorite.c.version == version,
+                favorite.c.user == user_id,
+                favorite.c.favKind == fav_kind,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def put_login_bonus(
         self, user_id: int, version: int, preset_id: int, **login_bonus_data
     ) -> Optional[int]:
@@ -390,7 +530,7 @@ class ChuniItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_user_gacha: Failed to insert! aime_id: {aime_id}")
+            self.logger.warning(f"put_user_gacha: Failed to insert! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -432,7 +572,7 @@ class ChuniItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_user_print_state: Failed to insert! aime_id: {aime_id}"
             )
             return None
@@ -449,7 +589,7 @@ class ChuniItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_user_print_detail: Failed to insert! aime_id: {aime_id}"
             )
             return None

titles/chuni/schema/profile.py

@@ -89,8 +89,6 @@ profile = Table(
         Integer,
         ForeignKey("chuni_profile_team.id", ondelete="SET NULL", onupdate="SET NULL"),
     ),
-    Column("avatarBack", Integer, server_default="0"),
-    Column("avatarFace", Integer, server_default="0"),
     Column("eliteRankPoint", Integer, server_default="0"),
     Column("stockedGridCount", Integer, server_default="0"),
     Column("netBattleLoseCount", Integer, server_default="0"),
@@ -98,10 +96,8 @@ profile = Table(
     Column("netBattle4thCount", Integer, server_default="0"),
     Column("overPowerRate", Integer, server_default="0"),
     Column("battleRewardStatus", Integer, server_default="0"),
-    Column("avatarPoint", Integer, server_default="0"),
     Column("netBattle1stCount", Integer, server_default="0"),
     Column("charaIllustId", Integer, server_default="0"),
-    Column("avatarItem", Integer, server_default="0"),
     Column("userNameEx", String(8), server_default=""),
     Column("netBattleWinCount", Integer, server_default="0"),
     Column("netBattleCorrection", Integer, server_default="0"),
@@ -112,7 +108,6 @@ profile = Table(
     Column("netBattle3rdCount", Integer, server_default="0"),
     Column("netBattleConsecutiveWinCount", Integer, server_default="0"),
     Column("overPowerLowerRank", Integer, server_default="0"),
-    Column("avatarWear", Integer, server_default="0"),
     Column("classEmblemBase", Integer, server_default="0"),
     Column("battleRankPoint", Integer, server_default="0"),
     Column("netBattle2ndCount", Integer, server_default="0"),
@@ -120,13 +115,19 @@ profile = Table(
     Column("skillId", Integer, server_default="0"),
     Column("lastCountryCode", String(5), server_default="JPN"),
     Column("isNetBattleHost", Boolean, server_default="0"),
-    Column("avatarFront", Integer, server_default="0"),
-    Column("avatarSkin", Integer, server_default="0"),
     Column("battleRewardCount", Integer, server_default="0"),
     Column("battleRewardIndex", Integer, server_default="0"),
     Column("netBattlePlayCount", Integer, server_default="0"),
     Column("exMapLoopCount", Integer, server_default="0"),
     Column("netBattleEndState", Integer, server_default="0"),
+    Column("rankUpChallengeResults", JSON),
+    Column("avatarBack", Integer, server_default="0"),
+    Column("avatarFace", Integer, server_default="0"),
+    Column("avatarPoint", Integer, server_default="0"),
+    Column("avatarItem", Integer, server_default="0"),
+    Column("avatarWear", Integer, server_default="0"),
+    Column("avatarFront", Integer, server_default="0"),
+    Column("avatarSkin", Integer, server_default="0"),
     Column("avatarHead", Integer, server_default="0"),
     UniqueConstraint("user", "version", name="chuni_profile_profile_uk"),
     mysql_charset="utf8mb4",
@@ -409,7 +410,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_profile_data: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_profile_data: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -417,8 +418,8 @@ class ChuniProfileData(BaseData):
         sql = (
             select([profile, option])
             .join(option, profile.c.user == option.c.user)
-            .filter(and_(profile.c.user == aime_id, profile.c.version == version))
+            .filter(and_(profile.c.user == aime_id, profile.c.version <= version))
-        )
+        ).order_by(profile.c.version.desc())
 
         result = self.execute(sql)
         if result is None:
@@ -429,9 +430,9 @@ class ChuniProfileData(BaseData):
         sql = select(profile).where(
             and_(
                 profile.c.user == aime_id,
-                profile.c.version == version,
+                profile.c.version <= version,
             )
-        )
+        ).order_by(profile.c.version.desc())
 
         result = self.execute(sql)
         if result is None:
@@ -451,7 +452,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_data_ex: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -461,9 +462,9 @@ class ChuniProfileData(BaseData):
         sql = select(profile_ex).where(
             and_(
                 profile_ex.c.user == aime_id,
-                profile_ex.c.version == version,
+                profile_ex.c.version <= version,
             )
-        )
+        ).order_by(profile_ex.c.version.desc())
 
         result = self.execute(sql)
         if result is None:
@@ -478,7 +479,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_option: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -502,7 +503,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_option_ex: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -526,7 +527,7 @@ class ChuniProfileData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_recent_rating: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -551,7 +552,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_activity: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -577,7 +578,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_charge: Failed to update! aime_id: {aime_id}"
             )
             return None

titles/chuni/schema/score.py

@@ -134,7 +134,9 @@ playlog = Table(
     Column("charaIllustId", Integer),
     Column("romVersion", String(255)),
     Column("judgeHeaven", Integer),
-    mysql_charset="utf8mb4",
+    Column("regionId", Integer),
+    Column("machineType", Integer),
+    mysql_charset="utf8mb4"
 )
 
 

titles/chuni/schema/static.py

@@ -1,11 +1,19 @@
 from typing import Dict, List, Optional
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
+from sqlalchemy import (
+    ForeignKeyConstraint,
+    Table,
+    Column,
+    UniqueConstraint,
+    PrimaryKeyConstraint,
+    and_,
+)
 from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, Float
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.engine import Row
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.sql import func, select
 from sqlalchemy.dialects.mysql import insert
+from datetime import datetime
 
 from core.data.schema import BaseData, metadata
 
@@ -17,6 +25,7 @@ events = Table(
     Column("eventId", Integer),
     Column("type", Integer),
     Column("name", String(255)),
+    Column("startDate", TIMESTAMP, server_default=func.now()),
     Column("enabled", Boolean, server_default="1"),
     UniqueConstraint("version", "eventId", name="chuni_static_events_uk"),
     mysql_charset="utf8mb4",
@@ -125,11 +134,13 @@ gacha_cards = Table(
 login_bonus_preset = Table(
     "chuni_static_login_bonus_preset",
     metadata,
-    Column("id", Integer, primary_key=True, nullable=False),
+    Column("presetId", Integer, nullable=False),
     Column("version", Integer, nullable=False),
     Column("presetName", String(255), nullable=False),
     Column("isEnabled", Boolean, server_default="1"),
-    UniqueConstraint("version", "id", name="chuni_static_login_bonus_preset_uk"),
+    PrimaryKeyConstraint(
+        "presetId", "version", name="chuni_static_login_bonus_preset_pk"
+    ),
     mysql_charset="utf8mb4",
 )
 
@@ -138,15 +149,7 @@ login_bonus = Table(
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
     Column("version", Integer, nullable=False),
-    Column(
+    Column("presetId", Integer, nullable=False),
-        "presetId",
-        ForeignKey(
-            "chuni_static_login_bonus_preset.id",
-            ondelete="cascade",
-            onupdate="cascade",
-        ),
-        nullable=False,
-    ),
     Column("loginBonusId", Integer, nullable=False),
     Column("loginBonusName", String(255), nullable=False),
     Column("presentId", Integer, nullable=False),
@@ -157,6 +160,16 @@ login_bonus = Table(
     UniqueConstraint(
         "version", "presetId", "loginBonusId", name="chuni_static_login_bonus_uk"
     ),
+    ForeignKeyConstraint(
+        ["presetId", "version"],
+        [
+            "chuni_static_login_bonus_preset.presetId",
+            "chuni_static_login_bonus_preset.version",
+        ],
+        onupdate="CASCADE",
+        ondelete="CASCADE",
+        name="chuni_static_login_bonus_ibfk_1",
+    ),
     mysql_charset="utf8mb4",
 )
 
@@ -200,7 +213,9 @@ class ChuniStaticData(BaseData):
         return result.lastrowid
 
     def get_login_bonus(
-        self, version: int, preset_id: int,
+        self,
+        version: int,
+        preset_id: int,
     ) -> Optional[List[Row]]:
         sql = login_bonus.select(
             and_(
@@ -234,7 +249,7 @@ class ChuniStaticData(BaseData):
         self, version: int, preset_id: int, preset_name: str, is_enabled: bool
     ) -> Optional[int]:
         sql = insert(login_bonus_preset).values(
-            id=preset_id,
+            presetId=preset_id,
             version=version,
             presetName=preset_name,
             isEnabled=is_enabled,
@@ -287,14 +302,14 @@ class ChuniStaticData(BaseData):
 
         result = self.execute(sql)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"update_event: failed to update event! version: {version}, event_id: {event_id}, enabled: {enabled}"
             )
             return None
 
         event = self.get_event(version, event_id)
         if event is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"update_event: failed to fetch event {event_id} after updating"
             )
             return None
@@ -414,6 +429,14 @@ class ChuniStaticData(BaseData):
             return None
         return result.fetchall()
 
+    def get_music(self, version: int) -> Optional[List[Row]]:
+        sql = music.select(music.c.version <= version)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def get_music_chart(
         self, version: int, song_id: int, chart_id: int
     ) -> Optional[List[Row]]:
@@ -483,7 +506,7 @@ class ChuniStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert gacha! gacha_id {gacha_id}")
+            self.logger.warning(f"Failed to insert gacha! gacha_id {gacha_id}")
             return None
         return result.lastrowid
 
@@ -518,7 +541,7 @@ class ChuniStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert gacha card! gacha_id {gacha_id}")
+            self.logger.warning(f"Failed to insert gacha card! gacha_id {gacha_id}")
             return None
         return result.lastrowid
 
@@ -554,7 +577,7 @@ class ChuniStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert card! card_id {card_id}")
+            self.logger.warning(f"Failed to insert card! card_id {card_id}")
             return None
         return result.lastrowid
 

titles/chuni/sun.py

@@ -0,0 +1,37 @@
+from typing import Dict, Any
+
+from core.config import CoreConfig
+from titles.chuni.newplus import ChuniNewPlus
+from titles.chuni.const import ChuniConstants
+from titles.chuni.config import ChuniConfig
+
+
+class ChuniSun(ChuniNewPlus):
+    def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
+        super().__init__(core_cfg, game_cfg)
+        self.version = ChuniConstants.VER_CHUNITHM_SUN
+
+    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = super().handle_get_game_setting_api_request(data)
+        ret["gameSetting"]["romVersion"] = self.game_cfg.version.version(self.version)["rom"]
+        ret["gameSetting"]["dataVersion"] = self.game_cfg.version.version(self.version)["data"]
+        ret["gameSetting"][
+            "matchingUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "matchingUriX"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "udpHolePunchUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "reflectorUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        return ret
+
+    def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
+        user_data = super().handle_cm_get_user_preview_api_request(data)
+
+        # hardcode lastDataVersion for CardMaker 1.35 A032
+        user_data["lastDataVersion"] = "2.10.00"
+        return user_data

titles/cm/base.py

@@ -23,19 +23,40 @@ class CardMakerBase:
         self.game = CardMakerConstants.GAME_CODE
         self.version = CardMakerConstants.VER_CARD_MAKER
 
+    @staticmethod
+    def _parse_int_ver(version: str) -> str:
+        return version.replace(".", "")[:3]
+
     def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
         if self.core_cfg.server.is_develop:
             uri = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}"
         else:
             uri = f"http://{self.core_cfg.title.hostname}"
 
-        # CHUNITHM = 0, maimai = 1, ONGEKI = 2
+        # grab the dict with all games version numbers from user config
+        games_ver = self.game_cfg.version.version(self.version)
+
         return {
             "length": 3,
             "gameConnectList": [
-                {"modelKind": 0, "type": 1, "titleUri": f"{uri}/SDHD/200/"},
+                # CHUNITHM
-                {"modelKind": 1, "type": 1, "titleUri": f"{uri}/SDEZ/120/"},
+                {
-                {"modelKind": 2, "type": 1, "titleUri": f"{uri}/SDDT/130/"},
+                    "modelKind": 0,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDHD/{self._parse_int_ver(games_ver['chuni'])}/",
+                },
+                # maimai DX
+                {
+                    "modelKind": 1,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDEZ/{self._parse_int_ver(games_ver['maimai'])}/",
+                },
+                # ONGEKI
+                {
+                    "modelKind": 2,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDDT/{self._parse_int_ver(games_ver['ongeki'])}/",
+                },
             ],
         }
 
@@ -47,12 +68,15 @@ class CardMakerBase:
             datetime.now() + timedelta(hours=4), self.date_time_format
         )
 
+        # grab the dict with all games version numbers from user config
+        games_ver = self.game_cfg.version.version(self.version)
+
         return {
             "gameSetting": {
                 "dataVersion": "1.30.00",
-                "ongekiCmVersion": "1.30.01",
+                "ongekiCmVersion": games_ver["ongeki"],
-                "chuniCmVersion": "2.00.00",
+                "chuniCmVersion": games_ver["chuni"],
-                "maimaiCmVersion": "1.20.00",
+                "maimaiCmVersion": games_ver["maimai"],
                 "requestInterval": 10,
                 "rebootStartTime": reboot_start,
                 "rebootEndTime": reboot_end,

titles/cm/cm135.py

@@ -1,8 +1,4 @@
-from datetime import date, datetime, timedelta
+from typing import Dict
-from typing import Any, Dict, List
-import json
-import logging
-from enum import Enum
 
 from core.config import CoreConfig
 from core.data.cache import cached
@@ -16,23 +12,7 @@ class CardMaker135(CardMakerBase):
         super().__init__(core_cfg, game_cfg)
         self.version = CardMakerConstants.VER_CARD_MAKER_135
 
-    def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_connect_api_request(data)
-        if self.core_cfg.server.is_develop:
-            uri = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}"
-        else:
-            uri = f"http://{self.core_cfg.title.hostname}"
-
-        ret["gameConnectList"][0]["titleUri"] = f"{uri}/SDHD/205/"
-        ret["gameConnectList"][1]["titleUri"] = f"{uri}/SDEZ/125/"
-        ret["gameConnectList"][2]["titleUri"] = f"{uri}/SDDT/135/"
-
-        return ret
-
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
         ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.35.00"
-        ret["gameSetting"]["ongekiCmVersion"] = "1.35.03"
-        ret["gameSetting"]["chuniCmVersion"] = "2.05.00"
-        ret["gameSetting"]["maimaiCmVersion"] = "1.25.00"
         return ret

titles/cm/config.py

@@ -1,3 +1,4 @@
+from typing import Dict
 from core.config import CoreConfig
 
 
@@ -20,6 +21,32 @@ class CardMakerServerConfig:
         )
 
 
+class CardMakerVersionConfig:
+    def __init__(self, parent_config: "CardMakerConfig") -> None:
+        self.__config = parent_config
+
+    def version(self, version: int) -> Dict:
+        """
+        in the form of:
+        1: {"ongeki": 1.30.01, "chuni": 2.00.00, "maimai": 1.20.00}
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "cardmaker", "version", default={
+                0: {    
+                    "ongeki": "1.30.01",
+                    "chuni": "2.00.00",
+                    "maimai": "1.20.00"
+                },
+                1: {
+                    "ongeki": "1.35.03",
+                    "chuni": "2.10.00",
+                    "maimai": "1.30.00"
+                }
+            }
+        )[version]
+
+
 class CardMakerConfig(dict):
     def __init__(self) -> None:
         self.server = CardMakerServerConfig(self)
+        self.version = CardMakerVersionConfig(self)

titles/cm/const.py

@@ -6,7 +6,7 @@ class CardMakerConstants:
     VER_CARD_MAKER = 0
     VER_CARD_MAKER_135 = 1
 
-    VERSION_NAMES = ("Card Maker 1.34", "Card Maker 1.35")
+    VERSION_NAMES = ("Card Maker 1.30", "Card Maker 1.35")
 
     @classmethod
     def game_ver_to_string(cls, ver: int):

titles/cm/index.py

@@ -30,7 +30,7 @@ class CardMakerServlet:
 
         self.versions = [
             CardMakerBase(core_cfg, self.game_cfg),
-            CardMaker135(core_cfg, self.game_cfg),
+            CardMaker135(core_cfg, self.game_cfg)
         ]
 
         self.logger = logging.getLogger("cardmaker")
@@ -85,11 +85,9 @@ class CardMakerServlet:
         endpoint = url_split[len(url_split) - 1]
         client_ip = Utils.get_ip_addr(request)
 
-        print(f"version: {version}")
-
         if version >= 130 and version < 135:  # Card Maker
             internal_ver = CardMakerConstants.VER_CARD_MAKER
-        elif version >= 135 and version < 136:  # Card Maker 1.35
+        elif version >= 135 and version < 140:  # Card Maker 1.35
             internal_ver = CardMakerConstants.VER_CARD_MAKER_135
 
         if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:
@@ -109,9 +107,7 @@ class CardMakerServlet:
 
         req_data = json.loads(unzip)
 
-        self.logger.info(
+        self.logger.info(f"v{version} {endpoint} request from {client_ip}")
-            f"v{version} {endpoint} request from {client_ip}"
-        )
         self.logger.debug(req_data)
 
         func_to_find = "handle_" + inflection.underscore(endpoint) + "_request"
@@ -126,11 +122,12 @@ class CardMakerServlet:
 
         except Exception as e:
             self.logger.error(f"Error handling v{version} method {endpoint} - {e}")
+            raise
             return zlib.compress(b'{"stat": "0"}')
 
         if resp is None:
             resp = {"returnCode": 1}
 
-        self.logger.info(f"Response {resp}")
+        self.logger.debug(f"Response {resp}")
 
         return zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8"))

titles/cm/read.py

@@ -68,7 +68,7 @@ class CardMakerReader(BaseReader):
                     read_csv = getattr(CardMakerReader, func)
                     read_csv(self, f"{self.bin_dir}/MU3/{file}")
                 else:
-                    self.logger.warn(
+                    self.logger.warning(
                         f"Couldn't find {file} file in {self.bin_dir}, skipping"
                     )
 
@@ -89,8 +89,7 @@ class CardMakerReader(BaseReader):
         version_ids = {
             "v2_00": ChuniConstants.VER_CHUNITHM_NEW,
             "v2_05": ChuniConstants.VER_CHUNITHM_NEW_PLUS,
-            # Chunithm SUN, ignore for now
+            "v2_10": ChuniConstants.VER_CHUNITHM_SUN,
-            "v2_10": ChuniConstants.VER_CHUNITHM_NEW_PLUS + 1
         }
 
         for root, dirs, files in os.walk(base_dir):
@@ -138,8 +137,7 @@ class CardMakerReader(BaseReader):
         version_ids = {
             "v2_00": ChuniConstants.VER_CHUNITHM_NEW,
             "v2_05": ChuniConstants.VER_CHUNITHM_NEW_PLUS,
-            # Chunithm SUN, ignore for now
+            "v2_10": ChuniConstants.VER_CHUNITHM_SUN,
-            "v2_10": ChuniConstants.VER_CHUNITHM_NEW_PLUS + 1,
         }
 
         for root, dirs, files in os.walk(base_dir):
@@ -226,6 +224,12 @@ class CardMakerReader(BaseReader):
                             True if troot.find("disable").text == "false" else False
                         )
 
+                        # check if a date is part of the name and disable the
+                        # card if it is
+                        enabled = (
+                            False if re.search(r"\d{2}/\d{2}/\d{2}", name) else enabled
+                        )
+
                         self.mai2_data.static.put_card(
                             version, card_id, name, enabled=enabled
                         )

titles/cxb/base.py

@@ -2,7 +2,7 @@ import logging
 import json
 from decimal import Decimal
 from base64 import b64encode
-from typing import Any, Dict
+from typing import Any, Dict, List
 from hashlib import md5
 from datetime import datetime
 
@@ -11,6 +11,7 @@ from titles.cxb.config import CxbConfig
 from titles.cxb.const import CxbConstants
 from titles.cxb.database import CxbData
 
+from threading import Thread
 
 class CxbBase:
     def __init__(self, cfg: CoreConfig, game_cfg: CxbConfig) -> None:
@@ -51,9 +52,154 @@ class CxbBase:
             self.logger.info(f"Login user {data['login']['authid']}")
             return {"token": data["login"]["authid"], "uid": data["login"]["authid"]}
 
-        self.logger.warn(f"User {data['login']['authid']} does not have a profile")
+        self.logger.warning(f"User {data['login']['authid']} does not have a profile")
         return {}
 
+    def task_generateCoupon(index, data1):
+        # Coupons
+        for i in range(500, 510):
+            index.append(str(i))
+            couponid = int(i) - 500
+            dataValue = [
+                {
+                    "couponId": str(couponid),
+                    "couponNum": "1",
+                    "couponLog": [],
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateShopListTitle(index, data1):
+        # ShopList_Title
+        for i in range(200000, 201451):
+            index.append(str(i))
+            shopid = int(i) - 200000
+            dataValue = [
+                {
+                    "shopId": shopid,
+                    "shopState": "2",
+                    "isDisable": "t",
+                    "isDeleted": "f",
+                    "isSpecialFlag": "f",
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateShopListIcon(index, data1):
+        # ShopList_Icon
+        for i in range(202000, 202264):
+            index.append(str(i))
+            shopid = int(i) - 200000
+            dataValue = [
+                {
+                    "shopId": shopid,
+                    "shopState": "2",
+                    "isDisable": "t",
+                    "isDeleted": "f",
+                    "isSpecialFlag": "f",
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateStories(index, data1):
+        # Stories
+        for i in range(900000, 900003):
+            index.append(str(i))
+            storyid = int(i) - 900000
+            dataValue = [
+                {
+                    "storyId": storyid,
+                    "unlockState1": ["t"] * 10,
+                    "unlockState2": ["t"] * 10,
+                    "unlockState3": ["t"] * 10,
+                    "unlockState4": ["t"] * 10,
+                    "unlockState5": ["t"] * 10,
+                    "unlockState6": ["t"] * 10,
+                    "unlockState7": ["t"] * 10,
+                    "unlockState8": ["t"] * 10,
+                    "unlockState9": ["t"] * 10,
+                    "unlockState10": ["t"] * 10,
+                    "unlockState11": ["t"] * 10,
+                    "unlockState12": ["t"] * 10,
+                    "unlockState13": ["t"] * 10,
+                    "unlockState14": ["t"] * 10,
+                    "unlockState15": ["t"] * 10,
+                    "unlockState16": ["t"] * 10,
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateScoreData(song, index, data1):
+        song_data = song["data"]
+        songCode = []
+
+        songCode.append(
+            {
+                "mcode": song_data["mcode"],
+                "musicState": song_data["musicState"],
+                "playCount": song_data["playCount"],
+                "totalScore": song_data["totalScore"],
+                "highScore": song_data["highScore"],
+                "everHighScore": song_data["everHighScore"]
+                if "everHighScore" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "clearRate": song_data["clearRate"],
+                "rankPoint": song_data["rankPoint"],
+                "normalCR": song_data["normalCR"]
+                if "normalCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "survivalCR": song_data["survivalCR"]
+                if "survivalCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "ultimateCR": song_data["ultimateCR"]
+                if "ultimateCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "nohopeCR": song_data["nohopeCR"]
+                if "nohopeCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "combo": song_data["combo"],
+                "coupleUserId": song_data["coupleUserId"],
+                "difficulty": song_data["difficulty"],
+                "isFullCombo": song_data["isFullCombo"],
+                "clearGaugeType": song_data["clearGaugeType"],
+                "fieldType": song_data["fieldType"],
+                "gameType": song_data["gameType"],
+                "grade": song_data["grade"],
+                "unlockState": song_data["unlockState"],
+                "extraState": song_data["extraState"],
+            }
+        )
+        index.append(song_data["index"])
+        data1.append(
+            b64encode(
+                bytes(json.dumps(songCode[0], separators=(",", ":")), "utf-8")
+            ).decode("utf-8")
+        )
+
+    def task_generateIndexData(versionindex):
+        try:
+            v_profile = self.data.profile.get_profile_index(0, uid, self.version)
+            v_profile_data = v_profile["data"]
+            versionindex.append(int(v_profile_data["appVersion"]))
+        except Exception:
+            versionindex.append("10400")
+
     def handle_action_loadrange_request(self, data: Dict) -> Dict:
         range_start = data["loadrange"]["range"][0]
         range_end = data["loadrange"]["range"][1]
@@ -107,146 +253,29 @@ class CxbBase:
         900000 = Stories
         """
 
-        # Coupons
+        # Async threads to generate the response
-        for i in range(500, 510):
+        thread_Coupon = Thread(target=CxbBase.task_generateCoupon(index, data1))
-            index.append(str(i))
+        thread_ShopListTitle = Thread(target=CxbBase.task_generateShopListTitle(index, data1))
-            couponid = int(i) - 500
+        thread_ShopListIcon = Thread(target=CxbBase.task_generateShopListIcon(index, data1))
-            dataValue = [
+        thread_Stories = Thread(target=CxbBase.task_generateStories(index, data1))
-                {
-                    "couponId": str(couponid),
-                    "couponNum": "1",
-                    "couponLog": [],
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
 
-        # ShopList_Title
+        thread_Coupon.start()
-        for i in range(200000, 201451):
+        thread_ShopListTitle.start()
-            index.append(str(i))
+        thread_ShopListIcon.start()
-            shopid = int(i) - 200000
+        thread_Stories.start()
-            dataValue = [
-                {
-                    "shopId": shopid,
-                    "shopState": "2",
-                    "isDisable": "t",
-                    "isDeleted": "f",
-                    "isSpecialFlag": "f",
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
 
-        # ShopList_Icon
+        thread_Coupon.join()
-        for i in range(202000, 202264):
+        thread_ShopListTitle.join()
-            index.append(str(i))
+        thread_ShopListIcon.join()
-            shopid = int(i) - 200000
+        thread_Stories.join()
-            dataValue = [
-                {
-                    "shopId": shopid,
-                    "shopState": "2",
-                    "isDisable": "t",
-                    "isDeleted": "f",
-                    "isSpecialFlag": "f",
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
-
-        # Stories
-        for i in range(900000, 900003):
-            index.append(str(i))
-            storyid = int(i) - 900000
-            dataValue = [
-                {
-                    "storyId": storyid,
-                    "unlockState1": ["t"] * 10,
-                    "unlockState2": ["t"] * 10,
-                    "unlockState3": ["t"] * 10,
-                    "unlockState4": ["t"] * 10,
-                    "unlockState5": ["t"] * 10,
-                    "unlockState6": ["t"] * 10,
-                    "unlockState7": ["t"] * 10,
-                    "unlockState8": ["t"] * 10,
-                    "unlockState9": ["t"] * 10,
-                    "unlockState10": ["t"] * 10,
-                    "unlockState11": ["t"] * 10,
-                    "unlockState12": ["t"] * 10,
-                    "unlockState13": ["t"] * 10,
-                    "unlockState14": ["t"] * 10,
-                    "unlockState15": ["t"] * 10,
-                    "unlockState16": ["t"] * 10,
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
 
         for song in songs:
-            song_data = song["data"]
+            thread_ScoreData = Thread(target=CxbBase.task_generateScoreData(song, index, data1))
-            songCode = []
+            thread_ScoreData.start()
-
-            songCode.append(
-                {
-                    "mcode": song_data["mcode"],
-                    "musicState": song_data["musicState"],
-                    "playCount": song_data["playCount"],
-                    "totalScore": song_data["totalScore"],
-                    "highScore": song_data["highScore"],
-                    "everHighScore": song_data["everHighScore"]
-                    if "everHighScore" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "clearRate": song_data["clearRate"],
-                    "rankPoint": song_data["rankPoint"],
-                    "normalCR": song_data["normalCR"]
-                    if "normalCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "survivalCR": song_data["survivalCR"]
-                    if "survivalCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "ultimateCR": song_data["ultimateCR"]
-                    if "ultimateCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "nohopeCR": song_data["nohopeCR"]
-                    if "nohopeCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "combo": song_data["combo"],
-                    "coupleUserId": song_data["coupleUserId"],
-                    "difficulty": song_data["difficulty"],
-                    "isFullCombo": song_data["isFullCombo"],
-                    "clearGaugeType": song_data["clearGaugeType"],
-                    "fieldType": song_data["fieldType"],
-                    "gameType": song_data["gameType"],
-                    "grade": song_data["grade"],
-                    "unlockState": song_data["unlockState"],
-                    "extraState": song_data["extraState"],
-                }
-            )
-            index.append(song_data["index"])
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(songCode[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
 
         for v in index:
-            try:
+            thread_IndexData = Thread(target=CxbBase.task_generateIndexData(versionindex))
-                v_profile = self.data.profile.get_profile_index(0, uid, self.version)
+            thread_IndexData.start()
-                v_profile_data = v_profile["data"]
-                versionindex.append(int(v_profile_data["appVersion"]))
-            except:
-                versionindex.append("10400")
 
         return {"index": index, "data": data1, "version": versionindex}
 
@@ -257,7 +286,7 @@ class CxbBase:
             # REV Omnimix Version Fetcher
             gameversion = data["saveindex"]["data"][0][2]
             self.logger.warning(f"Game Version is {gameversion}")
-        except:
+        except Exception:
             pass
 
         if "10205" in gameversion:
@@ -319,7 +348,7 @@ class CxbBase:
         # Sunrise
         try:
             profileIndex = save_data["index"].index("0")
-        except:
+        except Exception:
             return {"data": ""}  # Maybe
 
         profile = json.loads(save_data["data"][profileIndex])
@@ -416,7 +445,7 @@ class CxbBase:
         self.logger.info(f"Get best rankings for {uid}")
         p = self.data.score.get_best_rankings(uid)
 
-        rankList: list[Dict[str, Any]] = []
+        rankList: List[Dict[str, Any]] = []
 
         for rank in p:
             if rank["song_id"] is not None:
@@ -467,7 +496,7 @@ class CxbBase:
                         score=int(rid["sc"][0]),
                         clear=rid["clear"],
                     )
-                except:
+                except Exception:
                     self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),
@@ -485,7 +514,7 @@ class CxbBase:
                         score=int(rid["sc"][0]),
                         clear=0,
                     )
-                except:
+                except Exception:
                     self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),

titles/cxb/index.py

@@ -101,11 +101,9 @@ class CxbServlet(resource.Resource):
                     f"Ready on ports {self.game_cfg.server.port} & {self.game_cfg.server.port_secure}"
                 )
             else:
-                self.logger.info(
+                self.logger.info(f"Ready on port {self.game_cfg.server.port}")
-                    f"Ready on port {self.game_cfg.server.port}"
-                )
 
-    def render_POST(self, request: Request):
+    def render_POST(self, request: Request, version: int, endpoint: str):
         version = 0
         internal_ver = 0
         func_to_find = ""
@@ -125,13 +123,13 @@ class CxbServlet(resource.Resource):
                 )
 
             except Exception as f:
-                self.logger.warn(
+                self.logger.warning(
                     f"Error decoding json: {e} / {f} - {req_url} - {req_bytes}"
                 )
                 return b""
 
         if req_json == {}:
-            self.logger.warn(f"Empty json request to {req_url}")
+            self.logger.warning(f"Empty json request to {req_url}")
             return b""
 
         cmd = url_split[len(url_split) - 1]
@@ -142,7 +140,7 @@ class CxbServlet(resource.Resource):
                 not type(req_json["dldate"]) is dict
                 or "filetype" not in req_json["dldate"]
             ):
-                self.logger.warn(f"Malformed dldate request: {req_url} {req_json}")
+                self.logger.warning(f"Malformed dldate request: {req_url} {req_json}")
                 return b""
 
             filetype = req_json["dldate"]["filetype"]

titles/cxb/read.py

@@ -33,7 +33,7 @@ class CxbReader(BaseReader):
         pull_bin_ram = True
 
         if not path.exists(f"{self.bin_dir}"):
-            self.logger.warn(f"Couldn't find csv file in {self.bin_dir}, skipping")
+            self.logger.warning(f"Couldn't find csv file in {self.bin_dir}, skipping")
             pull_bin_ram = False
 
         if pull_bin_ram:
@@ -123,5 +123,5 @@ class CxbReader(BaseReader):
                             genre,
                             int(row["easy"].replace("Easy ", "").replace("N/A", "0")),
                         )
-        except:
+        except Exception:
-            self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")
+            self.logger.warning(f"Couldn't read csv file in {self.bin_dir}, skipping")

titles/diva/__init__.py

@@ -7,4 +7,4 @@ index = DivaServlet
 database = DivaData
 reader = DivaReader
 game_codes = [DivaConstants.GAME_CODE]
-current_schema_version = 1
+current_schema_version = 5

titles/diva/base.py

@@ -3,6 +3,7 @@ from typing import Any, List, Dict
 import logging
 import json
 import urllib
+from threading import Thread
 
 from core.config import CoreConfig
 from titles.diva.config import DivaConfig
@@ -266,16 +267,17 @@ class DivaBase:
     def handle_festa_info_request(self, data: Dict) -> Dict:
         encoded = "&"
         params = {
-            "fi_id": "1,-1",
+            "fi_id": "1,2",
-            "fi_name": f"{self.core_cfg.server.name} Opening,xxx",
+            "fi_name": f"{self.core_cfg.server.name} Opening,Project DIVA Festa",
-            "fi_kind": "0,0",
+            # 0=PINK, 1=GREEN
+            "fi_kind": "1,0",
             "fi_difficulty": "-1,-1",
             "fi_pv_id_lst": "ALL,ALL",
             "fi_attr": "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF,7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
-            "fi_add_vp": "20,0",
+            "fi_add_vp": "20,5",
-            "fi_mul_vp": "1,1",
+            "fi_mul_vp": "1,2",
-            "fi_st": "2022-06-17 17:00:00.0,2014-07-08 18:10:11.0",
+            "fi_st": "2019-01-01 00:00:00.0,2019-01-01 00:00:00.0",
-            "fi_et": "2029-01-01 10:00:00.0,2014-07-08 18:10:11.0",
+            "fi_et": "2029-01-01 00:00:00.0,2029-01-01 00:00:00.0",
             "fi_lut": "{self.time_lut}",
         }
 
@@ -401,10 +403,10 @@ class DivaBase:
             response += f"&lv_pnt={profile['lv_pnt']}"
             response += f"&vcld_pts={profile['vcld_pts']}"
             response += f"&skn_eqp={profile['use_pv_skn_eqp']}"
-            response += f"&btn_se_eqp={profile['use_pv_btn_se_eqp']}"
+            response += f"&btn_se_eqp={profile['btn_se_eqp']}"
-            response += f"&sld_se_eqp={profile['use_pv_sld_se_eqp']}"
+            response += f"&sld_se_eqp={profile['sld_se_eqp']}"
-            response += f"&chn_sld_se_eqp={profile['use_pv_chn_sld_se_eqp']}"
+            response += f"&chn_sld_se_eqp={profile['chn_sld_se_eqp']}"
-            response += f"&sldr_tch_se_eqp={profile['use_pv_sldr_tch_se_eqp']}"
+            response += f"&sldr_tch_se_eqp={profile['sldr_tch_se_eqp']}"
             response += f"&passwd_stat={profile['passwd_stat']}"
 
             # Store stuff to add to rework
@@ -478,6 +480,21 @@ class DivaBase:
         response += f"&dsp_clr_sts={profile['dsp_clr_sts']}"
         response += f"&rgo_sts={profile['rgo_sts']}"
 
+        # Contest progress
+        response += f"&cv_cid=-1,-1,-1,-1"
+        response += f"&cv_sc=-1,-1,-1,-1"
+        response += f"&cv_bv=-1,-1,-1,-1"
+        response += f"&cv_bv=-1,-1,-1,-1"
+        response += f"&cv_bf=-1,-1,-1,-1"
+
+        # Contest now playing id, return -1 if no current playing contest
+        response += f"&cnp_cid={profile['cnp_cid']}"
+        response += f"&cnp_val={profile['cnp_val']}"
+        # border can be 0=bronzem 1=silver, 2=gold
+        response += f"&cnp_rr={profile['cnp_rr']}"
+        # only show contest specifier if it is not empty
+        response += f"&cnp_sp={profile['cnp_sp']}" if profile["cnp_sp"] != "" else ""
+
         # To be fully fixed
         if "my_qst_id" not in profile:
             response += f"&my_qst_id=-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1"
@@ -488,7 +505,63 @@ class DivaBase:
 
         response += f"&my_qst_prgrs=0,0,0,0,0,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1"
         response += f"&my_qst_et=2022-06-19%2010%3A28%3A52.0,2022-06-19%2010%3A28%3A52.0,2022-06-19%2010%3A28%3A52.0,2100-01-01%2008%3A59%3A59.0,2100-01-01%2008%3A59%3A59.0,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx"
-        response += f"&clr_sts=0,0,0,0,0,0,0,0,56,52,35,6,6,3,1,0,0,0,0,0"
+
+        # define a helper class to store all counts for clear, great,
+        # excellent and perfect
+        class ClearSet:
+            def __init__(self):
+                self.clear = 0
+                self.great = 0
+                self.excellent = 0
+                self.perfect = 0
+
+        # create a dict to store the ClearSets per difficulty
+        clear_set_dict = {
+            0: ClearSet(),  # easy
+            1: ClearSet(),  # normal
+            2: ClearSet(),  # hard
+            3: ClearSet(),  # extreme
+            4: ClearSet(),  # exExtreme
+        }
+
+        # get clear status from user scores
+        pv_records = self.data.score.get_best_scores(data["pd_id"])
+        clear_status = "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0"
+
+        if pv_records is not None:
+            for score in pv_records:
+                if score["edition"] == 0:
+                    # cheap and standard both count to "clear"
+                    if score["clr_kind"] in {1, 2}:
+                        clear_set_dict[score["difficulty"]].clear += 1
+                    elif score["clr_kind"] == 3:
+                        clear_set_dict[score["difficulty"]].great += 1
+                    elif score["clr_kind"] == 4:
+                        clear_set_dict[score["difficulty"]].excellent += 1
+                    elif score["clr_kind"] == 5:
+                        clear_set_dict[score["difficulty"]].perfect += 1
+                else:
+                    # 4=ExExtreme
+                    if score["clr_kind"] in {1, 2}:
+                        clear_set_dict[4].clear += 1
+                    elif score["clr_kind"] == 3:
+                        clear_set_dict[4].great += 1
+                    elif score["clr_kind"] == 4:
+                        clear_set_dict[4].excellent += 1
+                    elif score["clr_kind"] == 5:
+                        clear_set_dict[4].perfect += 1
+
+            # now add all values to a list
+            clear_list = []
+            for clear_set in clear_set_dict.values():
+                clear_list.append(clear_set.clear)
+                clear_list.append(clear_set.great)
+                clear_list.append(clear_set.excellent)
+                clear_list.append(clear_set.perfect)
+
+            clear_status = ",".join(map(str, clear_list))
+
+        response += f"&clr_sts={clear_status}"
 
         # Store stuff to add to rework
         response += f"&mdl_eqp_tm={self.time_lut}"
@@ -591,50 +664,66 @@ class DivaBase:
 
         return pv_result
 
+    def task_generateScoreData(self, data: Dict, pd_by_pv_id, song):
+        
+        if int(song) > 0:
+            # the request do not send a edition so just perform a query best score and ranking for each edition.
+            # 0=ORIGINAL, 1=EXTRA
+            pd_db_song_0 = self.data.score.get_best_user_score(
+                data["pd_id"], int(song), data["difficulty"], edition=0
+            )
+            pd_db_song_1 = self.data.score.get_best_user_score(
+                data["pd_id"], int(song), data["difficulty"], edition=1
+            )
+
+            pd_db_ranking_0, pd_db_ranking_1 = None, None
+            if pd_db_song_0:
+                pd_db_ranking_0 = self.data.score.get_global_ranking(
+                    data["pd_id"], int(song), data["difficulty"], edition=0
+                )
+
+            if pd_db_song_1:
+                pd_db_ranking_1 = self.data.score.get_global_ranking(
+                    data["pd_id"], int(song), data["difficulty"], edition=1
+                )
+
+            pd_db_customize = self.data.pv_customize.get_pv_customize(
+                data["pd_id"], int(song)
+            )
+
+            # generate the pv_result string with the ORIGINAL edition and the EXTRA edition appended
+            pv_result = self._get_pv_pd_result(
+                int(song), pd_db_song_0, pd_db_ranking_0, pd_db_customize, edition=0
+            )
+            pv_result += "," + self._get_pv_pd_result(
+                int(song), pd_db_song_1, pd_db_ranking_1, pd_db_customize, edition=1
+            )
+
+            self.logger.debug(f"pv_result = {pv_result}")
+            pd_by_pv_id.append(urllib.parse.quote(pv_result))
+        else:
+            pd_by_pv_id.append(urllib.parse.quote(f"{song}***"))
+        pd_by_pv_id.append(",")
+
     def handle_get_pv_pd_request(self, data: Dict) -> Dict:
         song_id = data["pd_pv_id_lst"].split(",")
         pv = ""
 
+        threads = []
+        pd_by_pv_id = []
+
         for song in song_id:
-            if int(song) > 0:
+            thread_ScoreData = Thread(target=self.task_generateScoreData(data, pd_by_pv_id, song))
-                # the request do not send a edition so just perform a query best score and ranking for each edition.
+            threads.append(thread_ScoreData)
-                # 0=ORIGINAL, 1=EXTRA
-                pd_db_song_0 = self.data.score.get_best_user_score(
-                    data["pd_id"], int(song), data["difficulty"], edition=0
-                )
-                pd_db_song_1 = self.data.score.get_best_user_score(
-                    data["pd_id"], int(song), data["difficulty"], edition=1
-                )
 
-                pd_db_ranking_0, pd_db_ranking_1 = None, None
+        for x in threads:
-                if pd_db_song_0:
+            x.start()
-                    pd_db_ranking_0 = self.data.score.get_global_ranking(
-                        data["pd_id"], int(song), data["difficulty"], edition=0
-                    )
 
-                if pd_db_song_1:
+        for x in threads:
-                    pd_db_ranking_1 = self.data.score.get_global_ranking(
+            x.join()
-                        data["pd_id"], int(song), data["difficulty"], edition=1
-                    )
 
-                pd_db_customize = self.data.pv_customize.get_pv_customize(
+        for x in pd_by_pv_id:
-                    data["pd_id"], int(song)
+            pv += x
-                )
-
-                # generate the pv_result string with the ORIGINAL edition and the EXTRA edition appended
-                pv_result = self._get_pv_pd_result(
-                    int(song), pd_db_song_0, pd_db_ranking_0, pd_db_customize, edition=0
-                )
-                pv_result += "," + self._get_pv_pd_result(
-                    int(song), pd_db_song_1, pd_db_ranking_1, pd_db_customize, edition=1
-                )
-
-                self.logger.debug(f"pv_result = {pv_result}")
-
-                pv += urllib.parse.quote(pv_result)
-            else:
-                pv += urllib.parse.quote(f"{song}***")
-            pv += ","
 
         response = ""
         response += f"&pd_by_pv_id={pv[:-1]}"

titles/diva/read.py

@@ -34,18 +34,18 @@ class DivaReader(BaseReader):
         pull_opt_rom = True
 
         if not path.exists(f"{self.bin_dir}/ram"):
-            self.logger.warn(f"Couldn't find ram folder in {self.bin_dir}, skipping")
+            self.logger.warning(f"Couldn't find ram folder in {self.bin_dir}, skipping")
             pull_bin_ram = False
 
         if not path.exists(f"{self.bin_dir}/rom"):
-            self.logger.warn(f"Couldn't find rom folder in {self.bin_dir}, skipping")
+            self.logger.warning(f"Couldn't find rom folder in {self.bin_dir}, skipping")
             pull_bin_rom = False
 
         if self.opt_dir is not None:
             opt_dirs = self.get_data_directories(self.opt_dir)
         else:
             pull_opt_rom = False
-            self.logger.warn("No option directory specified, skipping")
+            self.logger.warning("No option directory specified, skipping")
 
         if pull_bin_ram:
             self.read_ram(f"{self.bin_dir}/ram")
@@ -139,7 +139,7 @@ class DivaReader(BaseReader):
                                 else:
                                     continue
         else:
-            self.logger.warn(f"Databank folder not found in {ram_root_dir}, skipping")
+            self.logger.warning(f"Databank folder not found in {ram_root_dir}, skipping")
 
     def read_rom(self, rom_root_dir: str) -> None:
         self.logger.info(f"Read ROM from {rom_root_dir}")
@@ -150,7 +150,7 @@ class DivaReader(BaseReader):
         elif path.exists(f"{rom_root_dir}/pv_db.txt"):
             file_path = f"{rom_root_dir}/pv_db.txt"
         else:
-            self.logger.warn(
+            self.logger.warning(
                 f"Cannot find pv_db.txt or mdata_pv_db.txt in {rom_root_dir}, skipping"
             )
             return

titles/diva/schema/profile.py

@@ -34,9 +34,17 @@ profile = Table(
     Column("use_pv_sld_se_eqp", Boolean, nullable=False, server_default="0"),
     Column("use_pv_chn_sld_se_eqp", Boolean, nullable=False, server_default="0"),
     Column("use_pv_sldr_tch_se_eqp", Boolean, nullable=False, server_default="0"),
+    Column("btn_se_eqp", Integer, nullable=False, server_default="-1"),
+    Column("sld_se_eqp", Integer, nullable=False, server_default="-1"),
+    Column("chn_sld_se_eqp", Integer, nullable=False, server_default="-1"),
+    Column("sldr_tch_se_eqp", Integer, nullable=False, server_default="-1"),
     Column("nxt_pv_id", Integer, nullable=False, server_default="708"),
     Column("nxt_dffclty", Integer, nullable=False, server_default="2"),
     Column("nxt_edtn", Integer, nullable=False, server_default="0"),
+    Column("cnp_cid", Integer, nullable=False, server_default="-1"),
+    Column("cnp_val", Integer, nullable=False, server_default="-1"),
+    Column("cnp_rr", Integer, nullable=False, server_default="-1"),
+    Column("cnp_sp", String(255), nullable=False, server_default=""),
     Column("dsp_clr_brdr", Integer, nullable=False, server_default="7"),
     Column("dsp_intrm_rnk", Integer, nullable=False, server_default="1"),
     Column("dsp_clr_sts", Integer, nullable=False, server_default="1"),

titles/diva/schema/score.py

@@ -3,6 +3,7 @@ from sqlalchemy.types import Integer, String, TIMESTAMP, JSON, Boolean
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.sql import func, select
 from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.engine import Row
 from typing import Optional, List, Dict, Any
 
 from core.data.schema import BaseData, metadata
@@ -167,7 +168,7 @@ class DivaScoreData(BaseData):
 
     def get_best_user_score(
         self, user_id: int, pv_id: int, difficulty: int, edition: int
-    ) -> Optional[Dict]:
+    ) -> Optional[Row]:
         sql = score.select(
             and_(
                 score.c.user == user_id,
@@ -184,7 +185,7 @@ class DivaScoreData(BaseData):
 
     def get_top3_scores(
         self, pv_id: int, difficulty: int, edition: int
-    ) -> Optional[List[Dict]]:
+    ) -> Optional[List[Row]]:
         sql = (
             score.select(
                 and_(
@@ -204,7 +205,7 @@ class DivaScoreData(BaseData):
 
     def get_global_ranking(
         self, user_id: int, pv_id: int, difficulty: int, edition: int
-    ) -> Optional[List]:
+    ) -> Optional[List[Row]]:
         # get the subquery max score of a user with pv_id, difficulty and
         # edition
         sql_sub = (
@@ -231,7 +232,7 @@ class DivaScoreData(BaseData):
             return None
         return result.fetchone()
 
-    def get_best_scores(self, user_id: int) -> Optional[List]:
+    def get_best_scores(self, user_id: int) -> Optional[List[Row]]:
         sql = score.select(score.c.user == user_id)
 
         result = self.execute(sql)

titles/idz/__init__.py

@@ -0,0 +1,8 @@
+from titles.idz.index import IDZServlet
+from titles.idz.const import IDZConstants
+from titles.idz.database import IDZData
+
+index = IDZServlet
+database = IDZData
+game_codes = [IDZConstants.GAME_CODE]
+current_schema_version = 1

titles/idz/config.py

@@ -0,0 +1,73 @@
+from typing import List, Dict
+
+from core.config import CoreConfig
+
+
+class IDZServerConfig:
+    def __init__(self, parent_config: "IDZConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "server", "enable", default=True
+        )
+
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "idz", "server", "loglevel", default="info"
+            )
+        )
+
+    @property
+    def hostname(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "server", "hostname", default=""
+        )
+
+    @property
+    def news(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "server", "news", default=""
+        )
+
+    @property
+    def aes_key(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "server", "aes_key", default=""
+        )
+
+
+class IDZPortsConfig:
+    def __init__(self, parent_config: "IDZConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def userdb(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "ports", "userdb", default=10000
+        )
+
+    @property
+    def match(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "ports", "match", default=10010
+        )
+
+    @property
+    def echo(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "ports", "echo", default=10020
+        )
+
+
+class IDZConfig(dict):
+    def __init__(self) -> None:
+        self.server = IDZServerConfig(self)
+        self.ports = IDZPortsConfig(self)
+
+    @property
+    def rsa_keys(self) -> List[Dict]:
+        return CoreConfig.get_config_field(self, "idz", "rsa_keys", default=[])

titles/idz/const.py

@@ -0,0 +1,53 @@
+from enum import Enum
+
+
+class IDZConstants:
+    GAME_CODE = "SDDF"
+
+    CONFIG_NAME = "idz.yaml"
+
+    VER_IDZ_110 = 0
+    VER_IDZ_130 = 1
+    VER_IDZ_210 = 2
+    VER_IDZ_230 = 3
+    NUM_VERS = 4
+
+    VERSION_NAMES = (
+        "Initial D Arcade Stage Zero v1.10",
+        "Initial D Arcade Stage Zero v1.30",
+        "Initial D Arcade Stage Zero v2.10",
+        "Initial D Arcade Stage Zero v2.30",
+    )
+
+    class PROFILE_STATUS(Enum):
+        LOCKED = 0
+        UNLOCKED = 1
+        OLD = 2
+
+    HASH_LUT = [
+        # No clue
+        0x9C82E674,
+        0x5A4738D9,
+        0x8B8D7AE0,
+        0x29EC9D81,
+        # These three are from AES TE0
+        0x1209091B,
+        0x1D83839E,
+        0x582C2C74,
+        0x341A1A2E,
+        0x361B1B2D,
+        0xDC6E6EB2,
+        0xB45A5AEE,
+        0x5BA0A0FB,
+        0xA45252F6,
+        0x763B3B4D,
+        0xB7D6D661,
+        0x7DB3B3CE,
+    ]
+    HASH_NUM = 0
+    HASH_MUL = [5, 7, 11, 12][HASH_NUM]
+    HASH_XOR = [0xB3, 0x8C, 0x14, 0x50][HASH_NUM]
+
+    @classmethod
+    def game_ver_to_string(cls, ver: int):
+        return cls.VERSION_NAMES[ver]

titles/idz/database.py

@@ -0,0 +1,7 @@
+from core.data import Data
+from core.config import CoreConfig
+
+
+class IDZData(Data):
+    def __init__(self, cfg: CoreConfig) -> None:
+        super().__init__(cfg)

titles/idz/echo.py

@@ -0,0 +1,19 @@
+from twisted.internet.protocol import DatagramProtocol
+import logging
+
+from core.config import CoreConfig
+from .config import IDZConfig
+
+
+class IDZEcho(DatagramProtocol):
+    def __init__(self, cfg: CoreConfig, game_cfg: IDZConfig) -> None:
+        super().__init__()
+        self.core_config = cfg
+        self.game_config = game_cfg
+        self.logger = logging.getLogger("idz")
+
+    def datagramReceived(self, data, addr):
+        self.logger.debug(
+            f"Echo from from {addr[0]}:{addr[1]} -> {self.transport.getHost().port} - {data.hex()}"
+        )
+        self.transport.write(data, addr)

titles/idz/handlers/__init__.py

@@ -0,0 +1,47 @@
+from .base import IDZHandlerBase
+
+from .load_server_info import IDZHandlerLoadServerInfo
+
+from .load_ghost import IDZHandlerLoadGhost
+
+from .load_config import IDZHandlerLoadConfigA, IDZHandlerLoadConfigB
+
+from .load_top_ten import IDZHandlerLoadTopTen
+
+from .update_story_clear_num import IDZHandlerUpdateStoryClearNum
+
+from .save_expedition import IDZHandlerSaveExpedition
+
+from .load_2on2 import IDZHandlerLoad2on2A, IDZHandlerLoad2on2B
+
+from .load_team_ranking import IDZHandlerLoadTeamRankingA, IDZHandlerLoadTeamRankingB
+
+from .discover_profile import IDZHandlerDiscoverProfile
+
+from .lock_profile import IDZHandlerLockProfile
+
+from .check_team_names import IDZHandlerCheckTeamName
+
+from .unknown import IDZHandlerUnknown
+
+from .create_profile import IDZHandlerCreateProfile
+
+from .create_auto_team import IDZHandlerCreateAutoTeam
+
+from .load_profile import IDZHandlerLoadProfile
+
+from .save_profile import IDZHandlerSaveProfile
+
+from .update_provisional_store_rank import IDZHandlerUpdateProvisionalStoreRank
+
+from .load_reward_table import IDZHandlerLoadRewardTable
+
+from .save_topic import IDZHandlerSaveTopic
+
+from .save_time_attack import IDZHandlerSaveTimeAttack
+
+from .unlock_profile import IDZHandlerUnlockProfile
+
+from .update_team_points import IDZHandleUpdateTeamPoints
+
+from .update_ui_report import IDZHandleUpdateUIReport

titles/idz/handlers/base.py

@@ -0,0 +1,26 @@
+import logging
+import struct
+from core.data import Data
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandlerBase:
+    name = "generic"
+    cmd_codes = [0x0000] * IDZConstants.NUM_VERS
+    rsp_codes = [0x0001] * IDZConstants.NUM_VERS
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        self.core_config = core_cfg
+        self.game_cfg = game_cfg
+        self.data = Data(core_cfg)
+        self.logger = logging.getLogger("idz")
+        self.game = IDZConstants.GAME_CODE
+        self.version = version
+        self.size = 0x30
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = bytearray([0] * self.size)
+        struct.pack_into("<H", ret, 0x0, self.rsp_codes[self.version])
+        return ret