Merge branch 'master' into fork_master

.gitattributes

@@ -0,0 +1,3 @@
+*.csv binary
+*.txt binary
+*.json binary

.gitignore

@@ -158,5 +158,6 @@ cert/*
 !cert/server.pem
 config/*
 deliver/*
+*.gz
 
 dbdump-*.json

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.9.15-slim-bullseye
+
+RUN apt update && apt install default-libmysqlclient-dev build-essential libtk nodejs npm pkg-config -y
+
+WORKDIR /app
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+RUN npm i -g nodemon
+
+COPY entrypoint.sh entrypoint.sh
+RUN chmod +x entrypoint.sh
+
+COPY index.py index.py
+COPY dbutils.py dbutils.py
+COPY read.py read.py
+ADD core core
+ADD titles titles
+ADD logs logs
+ADD cert cert
+
+ENTRYPOINT [ "/app/entrypoint.sh" ]

changelog.md

@@ -0,0 +1,194 @@
+# Changelog
+Documenting updates to ARTEMiS, to be updated every time the master branch is pushed to.
+
+## 20240109
+### System
++ Removed `ADD config config` from dockerfile
+
+### Aimedb
++ Fixed an error that resulted from trying to scan a banned or locked card
+
+## 20240108
+### System
++ Change how the underlying system handles URLs
+  + This can now allow for things like version-specific, or even keychip-specific URLs
+  + Specific changes to games are noted below
++ Fix docker files [#60](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/60) (Thanks Rylie!)
++ Fix support for python 3.8 - 3.10
+
+### Aimedb
++ Add support for SegaAuth key in games that support it (for now only Chunithm)
+  + This is a JWT that is sent to games, by Aimedb, that the games send to their game server, to verify that the access code the game is sending to the server was obtained via aimedb.
+  + Requires a base64-encoded secret to be set in the `core.yaml`
+
+### Chunithm
++ Fix Air support
++ Add saving for userRecentPlayerList
++ Add support for SegaAuthKey
++ Fix a bug arising if a user set their name to be 'true' or 'false'
++ Add support for Sun+ [#78](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/78) (Thanks EmmyHeart!)
++ Add `matching` section to `chuni.yaml`
++ ~~Change `udpHolePunchUri` and `reflectorUri` to be STUN and TURN servers~~ Reverted
++ Imrpove `GetGameSetting` request handling for different versions
++ Fix issue where songs would not always return all scores [#92](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/92) (Thanks Kumubou!)
+
+### maimai DX
++ Fix user charges failing to save
+
+### maimai
++ Made it functional
+
+### CXB
++ Improvements to request dispatching
++ Add support for non-omnimix music lists
+
+
+### IDZ
++ Fix news urls in accordance with the system change to URLs
+
+### Initial D THE ARCADE
++ Added support for Initial D THE ARCADE S2
+  + Story mode progress added
+  + Bunta Challenge/Touhou Project modes added
+  + Time Trials added
+  + Leaderboards added, but doesn't refresh sometimes
+  + Theory of Street mode added (with CPUs)
+  + Play Stamp/Timetrial events added
+  + Frontend to download profile added
+  + Importer to import profiles added
+
+### ONGEKI
++ Now supports HTTPS on a per-version basis
++ Merg PR [#61](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/61) (Thanks phantomlan!)
+  + Add Ranking Event Support
+  + Add reward list support
+  + Add version segregation to Event Ranking, Tech Challenge, and Music Ranking
+  + Now stores ClientTestmode and ClientSetting data
++ Fix mission points not adding correctly [#68](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/68) (Thanks phantomlan!)
++ Fix tech challenge [#70](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/70) (Thanks phantomlan!)
+
+### SAO
++ Change endpoint in accordance with the system change to URLs
++ Update request header class to be more accurate
++ Encrypted requests are now supported
++ Change to using handler classes instead of raw structs for simplicity
+
+### Wacca
++ Fix a server error causing a seperate error that casued issues
++ Add better error printing
++ Add better request validation
++ Fix HousingStartV2
++ Fix Lily's housing/get handler
+
+## 20231107
+### CXB
++  Hotfix `render_POST` sometimes failing to read the request body on large requests
+
+## 20231106
+### CXB
++ Hotfix `render_POST` function signature signature
++ Hotfix `handle_action_addenergy_request` hard failing if `get_energy` returns None
+
+## 20231015
+### maimai DX
++ Added support for FESTiVAL PLUS
+
+### Card Maker
++ Added support for maimai DX FESTiVAL PLUS
+
+## 20230716
+### General
++ Docker files added (#19)
++ Added support for threading
+  + This comes with the caviat that enabling it will not allow you to use Ctrl + C to stop the server.
+
+### Webui
++ Small improvements
++ Add card display
+
+### Allnet
++ Billing format validation
++ Fix naomitest.html endpoint
++ Add event logging for auths and billing
++ LoaderStateRecorder endpoint handler added
+
+### Mucha
++ Fixed log level always being "Info"
++ Add stub handler for DownloadState
+
+### Sword Art Online
++ Support added
+
+### Crossbeats
++ Added threading to profile loading
+  + This should cause a noticeable speed-up
+
+### Card Maker
++ DX Passes fixed
++ Various improvements
+
+### Diva
++ Added clear status calculation
++ Various minor fixes and improvements
+
+### Maimai
++ Added support for memorial photo uploads
++ Added support for the following versions
+  + Festival
+  + FiNALE
++ Various bug fixes and improvements
+
+### Wacca
++ Fixed an error that sometimes occoured when trying to unlock songs (#22)
+
+### Pokken
++ Profile saving added (loading TBA)
++ Use external STUN server for matching by default
+  + Matching still not working
+
+## 2023042300
+### Wacca
++ Time free now works properly
++ Fix reverse gate mission causing a fatal error
++ Other misc. fixes
++ Latest DB: 5
+
+### Pokken
++ Added preliminary support
+    + Nothing saves currently, but the game will boot and function properly.
+
+### Initial D Zero
++ Added preliminary support
+    + Nothing saves currently, but the game will boot and function for the most part.
+
+### Mai2
++ Added support for Festival
++ Lasted DB Version: 4
+
+### Ongeki
++ Misc fixes
++ Lasted DB Version: 4
+
+### Diva
++ Misc fixes
++ Lasted DB Version: 4
+
+### Chuni
++ Fix network encryption
++ Add `handle_remove_token_api_request` for event mode
+
+### Allnet
++ Added download order support
+    + It is up to the sysop to provide the INI file, and host the files.
+    + ONLY for use with cabs. It's not checked currently, which it's why it's default disabled
+    + YMMV, use at your own risk
++ When running develop mode, games that are not recognised will still be able to authenticate.
+
+### Database
++ Add autoupgrade command
+    + Invoke to automatically upgrade all schemas to their latest versions
+
++ `version` arg no longer required, leave it blank to update the game schema to latest if it isn't already
+
+### Misc
++ Update example nginx config file

contributing.md

@@ -0,0 +1,8 @@
+# Contributing to ARTEMiS
+If you would like to contribute to artemis, either by adding features, games, or fixing bugs, you can do so by forking the repo and submitting a pull request [here](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls). Please make sure, if you're submitting a PR for a game or game version, that you're following the n-0/y-1 guidelines, or it will be rejected.
+
+## Adding games
+Guide WIP
+
+## Adding game versions
+Guide WIP

core/__init__.py

@@ -4,3 +4,4 @@ from core.aimedb import AimedbFactory
 from core.title import TitleServlet
 from core.utils import Utils
 from core.mucha import MuchaServlet
+from core.frontend import FrontendServlet

core/adb_handlers/__init__.py

@@ -0,0 +1,6 @@
+from .base import ADBBaseRequest, ADBBaseResponse, ADBHeader, ADBHeaderException, PortalRegStatus, LogStatus, ADBStatus
+from .base import CompanyCodes, ReaderFwVer, CMD_CODE_GOODBYE, HEADER_SIZE
+from .lookup import ADBLookupRequest, ADBLookupResponse, ADBLookupExResponse
+from .campaign import ADBCampaignClearRequest, ADBCampaignClearResponse, ADBCampaignResponse, ADBOldCampaignRequest, ADBOldCampaignResponse
+from .felica import ADBFelicaLookupRequest, ADBFelicaLookupResponse, ADBFelicaLookup2Request, ADBFelicaLookup2Response
+from .log import ADBLogExRequest, ADBLogRequest, ADBStatusLogRequest, ADBLogExResponse

core/adb_handlers/base.py

@@ -0,0 +1,170 @@
+import struct
+from construct import Struct, Int16ul, Int32ul, PaddedString
+from enum import Enum
+import re
+from typing import Union, Final
+
+class LogStatus(Enum):
+    NONE = 0
+    START = 1
+    CONTINUE = 2
+    END = 3
+    OTHER = 4
+
+class PortalRegStatus(Enum):
+    NO_REG = 0
+    PORTAL = 1
+    SEGA_ID = 2
+
+class ADBStatus(Enum):
+    UNKNOWN = 0
+    GOOD = 1
+    BAD_AMIE_ID = 2
+    ALREADY_REG = 3
+    BAN_SYS_USER = 4
+    BAN_SYS = 5
+    BAN_USER = 6
+    BAN_GEN = 7
+    LOCK_SYS_USER = 8
+    LOCK_SYS = 9
+    LOCK_USER = 10
+
+class CompanyCodes(Enum):
+    NONE = 0
+    SEGA = 1
+    BAMCO = 2
+    KONAMI = 3
+    TAITO = 4
+
+class ReaderFwVer(Enum): # Newer readers use a singly byte value
+    NONE = 0
+    TN32_10 = 1
+    TN32_12 = 2
+    OTHER = 9
+    
+    def __str__(self) -> str:
+        if self == self.TN32_10:
+            return "TN32MSEC003S F/W Ver1.0"
+        elif self == self.TN32_12:
+            return "TN32MSEC003S F/W Ver1.2"
+        elif self == self.NONE:
+            return "Not Specified"
+        elif self == self.OTHER:
+            return "Unknown/Other"
+        else:
+            raise ValueError(f"Bad ReaderFwVer value {self.value}")
+    
+    @classmethod
+    def from_byte(self, byte: bytes) -> Union["ReaderFwVer", int]:
+        try:
+            i = int.from_bytes(byte, 'little')
+            try:
+                return ReaderFwVer(i)
+            except ValueError:
+                return i
+        except TypeError:
+            return 0
+
+class ADBHeaderException(Exception):
+    pass
+
+HEADER_SIZE: Final[int] = 0x20
+CMD_CODE_GOODBYE: Final[int] = 0x66
+
+# everything is LE
+class ADBHeader:
+    def __init__(self, magic: int, protocol_ver: int, cmd: int, length: int, status: int, game_id: Union[str, bytes], store_id: int, keychip_id: Union[str, bytes]) -> None:
+        self.magic = magic # u16
+        self.protocol_ver = protocol_ver # u16
+        self.cmd = cmd # u16
+        self.length = length # u16
+        try:
+            self.status = ADBStatus(status) # u16
+        except ValueError as e:
+            raise ADBHeaderException(f"Status is incorrect! {e}")
+        self.game_id = game_id # 4 char + \x00
+        self.store_id = store_id # u32
+        self.keychip_id = keychip_id# 11 char + \x00
+                
+        if type(self.game_id) == bytes:
+            self.game_id = self.game_id.decode()
+        
+        if type(self.keychip_id) == bytes:
+            self.keychip_id = self.keychip_id.decode()
+        
+        self.game_id = self.game_id.replace("\0", "")
+        self.keychip_id = self.keychip_id.replace("\0", "")
+        if self.cmd != CMD_CODE_GOODBYE: # Games for some reason send no data with goodbye
+            self.validate()
+    
+    @classmethod
+    def from_data(cls, data: bytes) -> "ADBHeader":
+        magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id = struct.unpack_from("<5H6sI12s", data)
+        head = cls(magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id)
+        
+        if head.length != len(data):
+            raise ADBHeaderException(f"Length is incorrect! Expect {head.length}, got {len(data)}")
+        
+        return head
+    
+    def validate(self) -> bool:
+        if self.magic != 0xa13e:
+            raise ADBHeaderException(f"Magic {self.magic} != 0xa13e")
+        
+        if self.protocol_ver < 0x1000:
+            raise ADBHeaderException(f"Protocol version {hex(self.protocol_ver)} is invalid!")
+
+        if re.fullmatch(r"^S[0-9A-Z]{3}[P]?$", self.game_id) is None:
+            raise ADBHeaderException(f"Game ID {self.game_id} is invalid!")
+        
+        if self.store_id == 0:
+            raise ADBHeaderException(f"Store ID cannot be 0!")
+
+        if re.fullmatch(r"^A[0-9]{2}[E|X][0-9]{2}[A-HJ-NP-Z][0-9]{4}$", self.keychip_id) is None:
+            raise ADBHeaderException(f"Keychip ID {self.keychip_id} is invalid!")
+        
+        return True
+
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "magic" / Int16ul,
+            "unknown" / Int16ul,
+            "response_code" / Int16ul,
+            "length" / Int16ul,
+            "status" / Int16ul,
+            "game_id" / PaddedString(6, 'utf_8'),
+            "store_id" / Int32ul,
+            "keychip_id" / PaddedString(12, 'utf_8'),
+        )
+
+        return resp_struct.build(dict(
+            magic=self.magic,
+            unknown=self.protocol_ver,
+            response_code=self.cmd,
+            length=self.length,
+            status=self.status.value,
+            game_id = self.game_id,
+            store_id = self.store_id,
+            keychip_id = self.keychip_id,
+        ))
+
+class ADBBaseRequest:
+    def __init__(self, data: bytes) -> None:
+        self.head = ADBHeader.from_data(data)
+
+class ADBBaseResponse:
+    def __init__(self, code: int = 0, length: int = 0x20, status: int = 1, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", protocol_ver: int = 0x3087) -> None:
+        self.head = ADBHeader(0xa13e, protocol_ver, code, length, status, game_id, store_id, keychip_id)
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, cmd: int, length: int = 0x20, status: int = 1) -> "ADBBaseResponse":
+        return cls(cmd, length, status, req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
+
+    def append_padding(self, data: bytes):
+        """Appends 0s to the end of the data until it's at the correct size"""
+        padding_size = self.head.length - len(data)
+        data += bytes(padding_size)
+        return data
+
+    def make(self) -> bytes:
+        return self.head.make()

core/adb_handlers/campaign.py

@@ -0,0 +1,132 @@
+from construct import Struct, Int16ul, Padding, Bytes, Int32ul, Int32sl
+
+from .base import *
+
+class Campaign:
+    def __init__(self) -> None:
+        self.id = 0
+        self.name = ""
+        self.announce_date = 0
+        self.start_date = 0
+        self.end_date = 0
+        self.distrib_start_date = 0
+        self.distrib_end_date = 0
+    
+    def make(self) -> bytes:
+        name_padding = bytes(128 - len(self.name))
+        return Struct(
+            "id" / Int32ul,
+            "name" / Bytes(128),
+            "announce_date" / Int32ul,
+            "start_date" / Int32ul,
+            "end_date" / Int32ul,
+            "distrib_start_date" / Int32ul,
+            "distrib_end_date" / Int32ul,
+            Padding(8),
+        ).build(dict(
+            id = self.id,
+            name = self.name.encode() + name_padding,
+            announce_date = self.announce_date,
+            start_date = self.start_date,
+            end_date = self.end_date,
+            distrib_start_date = self.distrib_start_date,
+            distrib_end_date = self.distrib_end_date,
+        ))
+
+class CampaignClear:
+    def __init__(self) -> None:
+        self.id = 0
+        self.entry_flag = 0
+        self.clear_flag = 0
+    
+    def make(self) -> bytes:
+        return Struct(
+            "id" / Int32ul,
+            "entry_flag" / Int32ul,
+            "clear_flag" / Int32ul,
+            Padding(4),
+        ).build(dict(
+            id = self.id,
+            entry_flag = self.entry_flag,
+            clear_flag = self.clear_flag,
+        ))
+
+class ADBCampaignResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0C, length: int = 0x200, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.campaigns = [Campaign(), Campaign(), Campaign()]
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self) -> bytes:
+        body = b""
+        
+        for c in self.campaigns:
+            body += c.make()
+        
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body
+
+class ADBOldCampaignRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.campaign_id = struct.unpack_from("<I", data, 0x20)
+
+class ADBOldCampaignResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0C, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.info0 = 0
+        self.info1 = 0
+        self.info2 = 0
+        self.info3 = 0
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "info0" / Int32sl,
+            "info1" / Int32sl,
+            "info2" / Int32sl,
+            "info3" / Int32sl,
+        ).build(
+            info0 = self.info0,
+            info1 = self.info1,
+            info2 = self.info2,
+            info3 = self.info3,
+        )
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+        return self.head.make() + resp_struct
+
+class ADBCampaignClearRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id = struct.unpack_from("<i", data, 0x20)
+
+class ADBCampaignClearResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0E, length: int = 0x50, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.campaign_clear_status = [CampaignClear(), CampaignClear(), CampaignClear()]
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:
+        body = b""
+        
+        for c in self.campaign_clear_status:
+            body += c.make()
+        
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/adb_handlers/felica.py

@@ -0,0 +1,85 @@
+from construct import Struct, Int32sl, Padding, Int8ub, Int16sl
+from typing import Union
+from .base import *
+
+class ADBFelicaLookupRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        idm, pmm = struct.unpack_from(">QQ", data, 0x20)
+        self.idm = hex(idm)[2:].upper()
+        self.pmm = hex(pmm)[2:].upper()
+
+class ADBFelicaLookupResponse(ADBBaseResponse):
+    def __init__(self, access_code: str = None, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x03, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.access_code = access_code if access_code is not None else "00000000000000000000"
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader, access_code: str = None) -> "ADBFelicaLookupResponse":
+        c = cls(access_code, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:        
+        resp_struct = Struct(
+            "felica_idx" / Int32ul,
+            "access_code" / Int8ub[10],
+            Padding(2)
+        ).build(dict(
+            felica_idx = 0,
+            access_code = bytes.fromhex(self.access_code)
+        ))
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+
+        return self.head.make() + resp_struct
+
+class ADBFelicaLookup2Request(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.random = struct.unpack_from("<16s", data, 0x20)[0]
+        idm, pmm = struct.unpack_from(">QQ", data, 0x30)
+        self.card_key_ver, self.write_ct, self.maca, company, fw_ver, self.dfc = struct.unpack_from("<16s16sQccH", data, 0x40)
+        self.idm = hex(idm)[2:].upper()
+        self.pmm = hex(pmm)[2:].upper()
+        self.company = CompanyCodes(int.from_bytes(company, 'little'))
+        self.fw_ver = ReaderFwVer.from_byte(fw_ver)
+
+class ADBFelicaLookup2Response(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None] = None, access_code: Union[str, None] = None, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x12, length: int = 0x130, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.access_code = access_code if access_code is not None else "00000000000000000000"
+        self.company = CompanyCodes.SEGA
+        self.portal_status = PortalRegStatus.NO_REG
+        self.auth_key = [0] * 256
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None] = None, access_code: Union[str, None] = None) -> "ADBFelicaLookup2Response":
+        c = cls(user_id, access_code, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self) -> bytes:        
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "relation1" / Int32sl,
+            "relation2" / Int32sl,
+            "access_code" / Int8ub[10],
+            "portal_status" / Int8ub,
+            "company_code" / Int8ub,
+            Padding(8),
+            "auth_key" / Int8ub[256],
+        ).build(dict(
+            user_id = self.user_id,
+            relation1 = -1, # Unsupported
+            relation2 = -1, # Unsupported
+            access_code = bytes.fromhex(self.access_code),
+            portal_status = self.portal_status.value,
+            company_code = self.company.value,
+            auth_key = self.auth_key
+        ))
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+
+        return self.head.make() + resp_struct

core/adb_handlers/log.py

@@ -0,0 +1,56 @@
+from construct import Struct, Padding, Int8sl
+from typing import Final, List
+
+from .base import *
+NUM_LOGS: Final[int] = 20
+NUM_LEN_LOG_EX: Final[int] = 48
+
+class AmLogEx:
+    def __init__(self, data: bytes) -> None:
+        self.aime_id, status, self.user_id, self.credit_ct, self.bet_ct, self.won_ct, self.local_time, \
+            self.tseq, self.place_id = struct.unpack("<IIQiii4xQiI", data)
+        self.status = LogStatus(status)
+
+class ADBStatusLogRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id, status = struct.unpack_from("<II", data, 0x20)
+        self.status = LogStatus(status)
+
+class ADBLogRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id, status, self.user_id, self.credit_ct, self.bet_ct, self.won_ct = struct.unpack_from("<IIQiii", data, 0x20)
+        self.status = LogStatus(status)
+
+class ADBLogExRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.logs: List[AmLogEx] = []
+
+        for x in range(NUM_LOGS):
+            self.logs.append(AmLogEx(data[0x20 + (NUM_LEN_LOG_EX * x): 0x50 + (NUM_LEN_LOG_EX * x)]))
+        
+        self.num_logs = struct.unpack_from("<I", data, 0x03E0)[0]
+
+class ADBLogExResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", protocol_ver: int = 12423, code: int = 20, length: int = 64, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id, protocol_ver)
+
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBLogExResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
+        return c
+    
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "log_result" / Int8sl[NUM_LOGS],
+            Padding(12)
+        )
+
+        body = resp_struct.build(dict(
+            log_result = [1] * NUM_LOGS
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/adb_handlers/lookup.py

@@ -0,0 +1,82 @@
+from construct import Struct, Int32sl, Padding, Int8sl
+from typing import Union
+
+from .base import *
+
+class ADBLookupException(Exception):
+    pass
+
+class ADBLookupRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.access_code = data[0x20:0x2A].hex()
+        company_code, fw_version, self.serial_number = struct.unpack_from("<bbI", data, 0x2A)
+        
+        try:
+            self.company_code = CompanyCodes(company_code)
+        except ValueError as e:
+            raise ADBLookupException(f"Invalid company code - {e}")
+        
+        self.fw_version = ReaderFwVer.from_byte(fw_version)
+
+class ADBLookupResponse(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None], game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x06, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.portal_reg = PortalRegStatus.NO_REG
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None]) -> "ADBLookupResponse":
+        c = cls(user_id, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self):
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "portal_reg" / Int8sl,
+            Padding(11)
+        )
+
+        body = resp_struct.build(dict(
+            user_id = self.user_id,
+            portal_reg = self.portal_reg.value
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body
+
+class ADBLookupExResponse(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None], game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", 
+                code: int = 0x10, length: int = 0x130, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.portal_reg = PortalRegStatus.NO_REG
+        self.auth_key = [0] * 256
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None]) -> "ADBLookupExResponse":
+        c = cls(user_id, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self):
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "portal_reg" / Int8sl,
+            Padding(3),
+            "auth_key" / Int8sl[256],
+            "relation1" / Int32sl,
+            "relation2" / Int32sl,
+        )
+
+        body = resp_struct.build(dict(
+            user_id = self.user_id,
+            portal_reg = self.portal_reg.value,
+            auth_key = self.auth_key,
+            relation1 = -1,
+            relation2 = -1
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/aimedb.py

@@ -2,26 +2,18 @@ from twisted.internet.protocol import Factory, Protocol
 import logging, coloredlogs
 from Crypto.Cipher import AES
 import struct
-from typing import Dict, Any
+from typing import Dict, Tuple, Callable, Union
+from typing_extensions import Final
 from logging.handlers import TimedRotatingFileHandler
 
 from core.config import CoreConfig
+from core.utils import create_sega_auth_key
 from core.data import Data
+from .adb_handlers import *
+
 
 class AimedbProtocol(Protocol):
-    AIMEDB_RESPONSE_CODES = {
+    request_list: Dict[int, Tuple[Callable[[bytes, int], Union[ADBBaseResponse, bytes]], int, str]] = {}
-        "felica_lookup": 0x03,
-        "lookup": 0x06,
-        "log": 0x0a,
-        "campaign": 0x0c,
-        "touch": 0x0e,
-        "lookup2": 0x10,
-        "felica_lookup2": 0x12,
-        "log2": 0x14,
-        "hello": 0x65
-    }
-
-    request_list: Dict[int, Any] = {}
 
     def __init__(self, core_cfg: CoreConfig) -> None:
         self.logger = logging.getLogger("aimedb")
@@ -31,16 +23,27 @@ class AimedbProtocol(Protocol):
             self.logger.error("!!!KEY NOT SET!!!")
             exit(1)
 
-        self.request_list[0x01] = self.handle_felica_lookup
+        self.register_handler(0x01, 0x03, self.handle_felica_lookup, 'felica_lookup')
-        self.request_list[0x04] = self.handle_lookup
+        self.register_handler(0x02, 0x03, self.handle_felica_register, 'felica_register')
-        self.request_list[0x05] = self.handle_register
+
-        self.request_list[0x09] = self.handle_log        
+        self.register_handler(0x04, 0x06, self.handle_lookup, 'lookup')
-        self.request_list[0x0b] = self.handle_campaign
+        self.register_handler(0x05, 0x06, self.handle_register, 'register')
-        self.request_list[0x0d] = self.handle_touch
+
-        self.request_list[0x0f] = self.handle_lookup2
+        self.register_handler(0x07, 0x08, self.handle_status_log, 'status_log')
-        self.request_list[0x11] = self.handle_felica_lookup2
+        self.register_handler(0x09, 0x0A, self.handle_log, 'aime_log')       
-        self.request_list[0x13] = self.handle_log2
+
-        self.request_list[0x64] = self.handle_hello
+        self.register_handler(0x0B, 0x0C, self.handle_campaign, 'campaign')
+        self.register_handler(0x0D, 0x0E, self.handle_campaign_clear, 'campaign_clear')
+
+        self.register_handler(0x0F, 0x10, self.handle_lookup_ex, 'lookup_ex')
+        self.register_handler(0x11, 0x12, self.handle_felica_lookup_ex, 'felica_lookup_ex')
+
+        self.register_handler(0x13, 0x14, self.handle_log_ex, 'aime_log_ex')
+        self.register_handler(0x64, 0x65, self.handle_hello, 'hello')
+        self.register_handler(0x66, 0, self.handle_goodbye, 'goodbye')
+    
+    def register_handler(self, cmd: int, resp:int, handler: Callable[[bytes, int], Union[ADBBaseResponse, bytes]], name: str) -> None:
+        self.request_list[cmd] = (handler, resp, name)
 
     def append_padding(self, data: bytes):
         """Appends 0s to the end of the data until it's at the correct size"""
@@ -53,166 +56,293 @@ class AimedbProtocol(Protocol):
         self.logger.debug(f"{self.transport.getPeer().host} Connected")
 
     def connectionLost(self, reason) -> None:
-        self.logger.debug(f"{self.transport.getPeer().host} Disconnected - {reason.value}")
+        self.logger.debug(
+            f"{self.transport.getPeer().host} Disconnected - {reason.value}"
+        )
 
     def dataReceived(self, data: bytes) -> None:
         cipher = AES.new(self.config.aimedb.key.encode(), AES.MODE_ECB)
 
         try:
             decrypted = cipher.decrypt(data)
-        except:
+        
-            self.logger.error(f"Failed to decrypt {data.hex()}")
+        except Exception as e:
+            self.logger.error(f"Failed to decrypt {data.hex()} because {e}")
             return None
 
         self.logger.debug(f"{self.transport.getPeer().host} wrote {decrypted.hex()}")
 
-        if not decrypted[1] == 0xa1 and not decrypted[0] == 0x3e:
+        try:
-            self.logger.error(f"Bad magic")
+            head = ADBHeader.from_data(decrypted)
-            return None
         
-        req_code = decrypted[4]
+        except ADBHeaderException as e:
+            self.logger.error(f"Error parsing ADB header: {e}")
+            try:    
+                encrypted = cipher.encrypt(ADBBaseResponse().make())
+                self.transport.write(encrypted)
+
+            except Exception as e:
+                self.logger.error(f"Failed to encrypt default response because {e}")
             
-        if req_code == 0x66:
-            self.logger.info(f"goodbye from {self.transport.getPeer().host}")
-            self.transport.loseConnection()
             return
 
+        if head.keychip_id == "ABCD1234567" or head.store_id == 0xfff0:
+            self.logger.warning(f"Request from uninitialized AMLib: {vars(head)}")
+
+        handler, resp_code, name = self.request_list.get(head.cmd, (self.handle_default, None, 'default'))
+
+        if resp_code is None:
+            self.logger.warning(f"No handler for cmd {hex(head.cmd)}")
+        
+        elif resp_code > 0:
+            self.logger.info(f"{name} from {head.keychip_id} ({head.game_id}) @ {self.transport.getPeer().host}")
+        
+        resp = handler(decrypted, resp_code)
+
+        if type(resp) == ADBBaseResponse or issubclass(type(resp), ADBBaseResponse):
+            resp_bytes = resp.make()
+            if len(resp_bytes) != resp.head.length:
+                resp_bytes = self.append_padding(resp_bytes)
+
+        elif type(resp) == bytes:
+            resp_bytes = resp
+        
+        elif resp is None: # Nothing to send, probably a goodbye
+            return
+        
+        else:
+            raise TypeError(f"Unsupported type returned by ADB handler for {name}: {type(resp)}")
+
         try:    
-            resp = self.request_list[req_code](decrypted)
+            encrypted = cipher.encrypt(resp_bytes)
-            encrypted = cipher.encrypt(resp)
+            self.logger.debug(f"Response {resp_bytes.hex()}")
-            self.logger.debug(f"Response {resp.hex()}")
             self.transport.write(encrypted)
 
-        except KeyError:
+        except Exception as e:
-            self.logger.error(f"Unknown command code {hex(req_code)}")
+            self.logger.error(f"Failed to encrypt {resp_bytes.hex()} because {e}")
-            return None
         
-        except ValueError as e:
+    def handle_default(self, data: bytes, resp_code: int, length: int = 0x20) -> ADBBaseResponse:
-            self.logger.error(f"Failed to encrypt {resp.hex()} because {e}")
+        req = ADBHeader.from_data(data)
-            return None
+        return ADBBaseResponse(resp_code, length, 1, req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
 
-    def handle_campaign(self, data: bytes) -> bytes:
+    def handle_hello(self, data: bytes, resp_code: int) -> ADBBaseResponse:
-        self.logger.info(f"campaign from {self.transport.getPeer().host}")
+        return self.handle_default(data, resp_code)
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["campaign"], 0x0200, 0x0001)
-        return self.append_padding(ret)
 
-    def handle_hello(self, data: bytes) -> bytes:
+    def handle_campaign(self, data: bytes, resp_code: int) -> ADBBaseResponse:
-        self.logger.info(f"hello from {self.transport.getPeer().host}")
+        h = ADBHeader.from_data(data)
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["hello"], 0x0020, 0x0001)
+        if h.protocol_ver >= 0x3030:
-        return self.append_padding(ret)
+            req = h
+            resp = ADBCampaignResponse.from_req(req)
 
-    def handle_lookup(self, data: bytes) -> bytes:
+        else:
-        luid = data[0x20: 0x2a].hex()
+            req = ADBOldCampaignRequest(data)
-        user_id = self.data.card.get_user_id_from_card(access_code=luid)
             
-        if user_id is None: user_id = -1
+            self.logger.info(f"Legacy campaign request for campaign {req.campaign_id} (protocol version {hex(h.protocol_ver)})")
+            resp = ADBOldCampaignResponse.from_req(req.head)
         
-        self.logger.info(f"lookup from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}")
+        # We don't currently support campaigns
+        return resp
 
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["lookup"], 0x0130, 0x0001)
+    def handle_lookup(self, data: bytes, resp_code: int) -> ADBBaseResponse:
-        ret += bytes(0x20 - len(ret))
+        req = ADBLookupRequest(data)
+        user_id = self.data.card.get_user_id_from_card(req.access_code)
+        is_banned = self.data.card.get_card_banned(req.access_code)
+        is_locked = self.data.card.get_card_locked(req.access_code)
 
-        if user_id is None: ret += struct.pack("<iH", -1, 0)
+        ret = ADBLookupResponse.from_req(req.head, user_id)
-        else: ret += struct.pack("<l", user_id)
+        if is_banned and is_locked:
-        return self.append_padding(ret)
+            ret.head.status = ADBStatus.BAN_SYS_USER
+        elif is_banned:
+            ret.head.status = ADBStatus.BAN_SYS
+        elif is_locked:
+            ret.head.status = ADBStatus.LOCK_USER
         
-    def handle_lookup2(self, data: bytes) -> bytes:
+        self.logger.info(
-        self.logger.info(f"lookup2")
+            f"access_code {req.access_code} -> user_id {ret.user_id}"
+        )
+        return ret
 
-        ret = bytearray(self.handle_lookup(data))
+    def handle_lookup_ex(self, data: bytes, resp_code: int) -> ADBBaseResponse:
-        ret[4] = self.AIMEDB_RESPONSE_CODES["lookup2"]
+        req = ADBLookupRequest(data)
+        user_id = self.data.card.get_user_id_from_card(req.access_code)
 
-        return bytes(ret)
+        is_banned = self.data.card.get_card_banned(req.access_code)
+        is_locked = self.data.card.get_card_locked(req.access_code)
 
-    def handle_felica_lookup(self, data: bytes) -> bytes:
+        ret = ADBLookupExResponse.from_req(req.head, user_id)
-        idm = data[0x20: 0x28].hex()
+        if is_banned and is_locked:
-        pmm = data[0x28: 0x30].hex()
+            ret.head.status = ADBStatus.BAN_SYS_USER
-        access_code = self.data.card.to_access_code(idm)
+        elif is_banned:
-        self.logger.info(f"felica_lookup from {self.transport.getPeer().host}: idm {idm} pmm {pmm} -> access_code {access_code}")
+            ret.head.status = ADBStatus.BAN_SYS
+        elif is_locked:
+            ret.head.status = ADBStatus.LOCK_USER
 
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["felica_lookup"], 0x0030, 0x0001)
+        self.logger.info(
-        ret += bytes(26)
+            f"access_code {req.access_code} -> user_id {ret.user_id}"
-        ret += bytes.fromhex(access_code)
+        )
 
-        return self.append_padding(ret)
+        if user_id and user_id > 0 and self.config.aimedb.id_secret:
+            auth_key = create_sega_auth_key(user_id, req.head.game_id, req.head.store_id, req.head.keychip_id, self.config.aimedb.id_secret, self.config.aimedb.id_lifetime_seconds)
+            if auth_key is not None:
+                auth_key_extra_len = 256 - len(auth_key)
+                auth_key_full = auth_key.encode() + (b"\0" * auth_key_extra_len)
+                self.logger.debug(f"Generated auth token {auth_key}")
+                ret.auth_key = auth_key_full
 
-    def handle_felica_lookup2(self, data: bytes) -> bytes:
+        return ret
-        idm = data[0x30: 0x38].hex()
-        pmm = data[0x38: 0x40].hex()
-        access_code = self.data.card.to_access_code(idm)
-        user_id = self.data.card.get_user_id_from_card(access_code=access_code)
 
-        if user_id is None: user_id = -1
+    def handle_felica_lookup(self, data: bytes, resp_code: int) -> bytes:
+        """
+        On official, I think a card has to be registered for this to actually work, but 
+        I'm making the executive decision to not implement that and just kick back our
+        faux generated access code. The real felica IDm -> access code conversion is done
+        on the ADB server, which we do not and will not ever have access to. Because we can
+        assure that all IDms will be unique, this basic 0-padded hex -> int conversion will
+        be fine.
+        """
+        req = ADBFelicaLookupRequest(data)
+        ac = self.data.card.to_access_code(req.idm)
+        self.logger.info(
+            f"idm {req.idm} ipm {req.pmm} -> access_code {ac}"
+        )
+        return ADBFelicaLookupResponse.from_req(req.head, ac)
 
-        self.logger.info(f"felica_lookup2 from {self.transport.getPeer().host}: idm {idm} ipm {pmm} -> access_code {access_code} user_id {user_id}")
+    def handle_felica_register(self, data: bytes, resp_code: int) -> bytes:
+        """
+        I've never seen this used.
+        """
+        req = ADBFelicaLookupRequest(data)
+        ac = self.data.card.to_access_code(req.idm)
         
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["felica_lookup2"], 0x0140, 0x0001)
+        if self.config.server.allow_user_registration:
-        ret += bytes(22)
-        ret += struct.pack("<lq", user_id, -1) # first -1 is ext_id, 3rd is access code
-        ret += bytes.fromhex(access_code)
-        ret += struct.pack("<l", 1)
-        
-        return self.append_padding(ret)
-    
-    def handle_touch(self, data: bytes) -> bytes:
-        self.logger.info(f"touch from {self.transport.getPeer().host}")
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["touch"], 0x0050, 0x0001)
-        ret += bytes(5)
-        ret += struct.pack("<3H", 0x6f, 0, 1)
-
-        return self.append_padding(ret)
-
-    def handle_register(self, data: bytes) -> bytes:        
-        luid = data[0x20: 0x2a].hex()
-        if self.config.server.allow_registration:
             user_id = self.data.user.create_user()
 
             if user_id is None:
-                user_id = -1
                 self.logger.error("Failed to register user!")
+                user_id = -1
 
             else:
-                card_id = self.data.card.create_card(user_id, luid)
+                card_id = self.data.card.create_card(user_id, ac)
 
                 if card_id is None:
-                    user_id = -1
                     self.logger.error("Failed to register card!")
+                    user_id = -1
 
-            self.logger.info(f"register from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}")
+            self.logger.info(
+                f"Register access code {ac} (IDm: {req.idm} PMm: {req.pmm}) -> user_id {user_id}"
+            )
 
         else:
-            self.logger.info(f"register from {self.transport.getPeer().host} blocked!: luid {luid}")
+            self.logger.info(
+                f"Registration blocked!: access code {ac} (IDm: {req.idm} PMm: {req.pmm})"
+            )
+
+        return ADBFelicaLookupResponse.from_req(req.head, ac)
+
+    def handle_felica_lookup_ex(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBFelicaLookup2Request(data)
+        access_code = self.data.card.to_access_code(req.idm)
+        user_id = self.data.card.get_user_id_from_card(access_code=access_code)
+
+        if user_id is None:
             user_id = -1
 
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["lookup"], 0x0030, 0x0001 if user_id > -1 else 0)
+        self.logger.info(
-        ret += bytes(0x20 - len(ret))
+            f"idm {req.idm} ipm {req.pmm} -> access_code {access_code} user_id {user_id}"
-        ret += struct.pack("<l", user_id)
+        )
 
-        return self.append_padding(ret)
+        resp = ADBFelicaLookup2Response.from_req(req.head, user_id, access_code)
 
-    def handle_log(self, data: bytes) -> bytes:
+        if user_id and user_id > 0 and self.config.aimedb.id_secret:
-        # TODO: Save aimedb logs
+            auth_key = create_sega_auth_key(user_id, req.head.game_id, req.head.store_id, req.head.keychip_id, self.config.aimedb.id_secret, self.config.aimedb.id_lifetime_seconds)
-        self.logger.info(f"log from {self.transport.getPeer().host}")
+            if auth_key is not None:
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["log"], 0x0020, 0x0001)
+                auth_key_extra_len = 256 - len(auth_key)
-        return self.append_padding(ret)
+                auth_key_full = auth_key.encode() + (b"\0" * auth_key_extra_len)
+                self.logger.debug(f"Generated auth token {auth_key}")
+                resp.auth_key = auth_key_full
 
-    def handle_log2(self, data: bytes) -> bytes:
+        return resp
-        self.logger.info(f"log2 from {self.transport.getPeer().host}")
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["log2"], 0x0040, 0x0001)
-        ret += bytes(22)
-        ret += struct.pack("H", 1)
 
-        return self.append_padding(ret)
+    def handle_campaign_clear(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        req = ADBCampaignClearRequest(data)
+
+        resp = ADBCampaignClearResponse.from_req(req.head)
+
+        # We don't support campaign stuff
+        return resp
+
+    def handle_register(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBLookupRequest(data)
+        user_id = -1
+
+        if self.config.server.allow_user_registration:
+            user_id = self.data.user.create_user()
+
+            if user_id is None:
+                self.logger.error("Failed to register user!")
+                user_id = -1
+
+            else:
+                card_id = self.data.card.create_card(user_id, req.access_code)
+
+                if card_id is None:
+                    self.logger.error("Failed to register card!")
+                    user_id = -1
+
+            self.logger.info(
+                f"Register access code {req.access_code} -> user_id {user_id}"
+            )
+
+        else:
+            self.logger.info(
+                f"Registration blocked!: access code {req.access_code}"
+            )
+
+        resp = ADBLookupResponse.from_req(req.head, user_id)
+        if resp.user_id <= 0:
+            resp.head.status = ADBStatus.BAN_SYS # Closest we can get to a "You cannot register"
+
+        return resp
+
+    # TODO: Save these in some capacity, as deemed relevant
+    def handle_status_log(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBStatusLogRequest(data)
+        self.logger.info(f"User {req.aime_id} logged {req.status.name} event")
+        return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
+
+    def handle_log(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBLogRequest(data)
+        self.logger.info(f"User {req.aime_id} logged {req.status.name} event, credit_ct: {req.credit_ct} bet_ct: {req.bet_ct} won_ct: {req.won_ct}")
+        return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
+
+    def handle_log_ex(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBLogExRequest(data)
+        strs = []
+        self.logger.info(f"Recieved {req.num_logs} or {len(req.logs)} logs")
+        
+        for x in range(req.num_logs):
+            self.logger.debug(f"User {req.logs[x].aime_id} logged {req.logs[x].status.name} event, credit_ct: {req.logs[x].credit_ct} bet_ct: {req.logs[x].bet_ct} won_ct: {req.logs[x].won_ct}")
+        return ADBLogExResponse.from_req(req.head)
+
+    def handle_goodbye(self, data: bytes, resp_code: int) -> None:
+        self.logger.info(f"goodbye from {self.transport.getPeer().host}")
+        self.transport.loseConnection()
+        return
 
 class AimedbFactory(Factory):
     protocol = AimedbProtocol
+
     def __init__(self, cfg: CoreConfig) -> None:
         self.config = cfg
         log_fmt_str = "[%(asctime)s] Aimedb | %(levelname)s | %(message)s"
         log_fmt = logging.Formatter(log_fmt_str)
         self.logger = logging.getLogger("aimedb")
 
-        fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.log_dir, "aimedb"), when="d", backupCount=10)
+        fileHandler = TimedRotatingFileHandler(
+            "{0}/{1}.log".format(self.config.server.log_dir, "aimedb"),
+            when="d",
+            backupCount=10,
+        )
         fileHandler.setFormatter(log_fmt)
 
         consoleHandler = logging.StreamHandler()
@@ -222,7 +352,9 @@ class AimedbFactory(Factory):
         self.logger.addHandler(consoleHandler)
 
         self.logger.setLevel(self.config.aimedb.loglevel)
-        coloredlogs.install(level=cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str)
+        coloredlogs.install(
+            level=cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str
+        )
 
         if self.config.aimedb.key == "":
             self.logger.error("Please set 'key' field in your config file.")

core/config.py

@@ -1,33 +1,71 @@
 import logging, os
 from typing import Any
 
+
 class ServerConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
     def listen_address(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'listen_address', default='127.0.0.1')
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "listen_address", default="127.0.0.1"
+        )
 
     @property
     def allow_user_registration(self) -> bool:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'allow_user_registration', default=True)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "allow_user_registration", default=True
+        )
 
     @property
     def allow_unregistered_serials(self) -> bool:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'allow_unregistered_serials', default=True)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "allow_unregistered_serials", default=True
+        )
 
     @property
     def name(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'name', default="ARTEMiS")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "name", default="ARTEMiS"
+        )
 
     @property
     def is_develop(self) -> bool:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'is_develop', default=True)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "is_develop", default=True
+        )
+
+    @property
+    def is_using_proxy(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "is_using_proxy", default=False
+        )
+
+    @property
+    def threading(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "threading", default=False
+        )
 
     @property
     def log_dir(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'log_dir', default='logs')
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "log_dir", default="logs"
+        )
+
+    @property
+    def check_arcade_ip(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "check_arcade_ip", default=False
+        )
+
+    @property
+    def strict_ip_checking(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "strict_ip_checking", default=False
+        )
+
 
 class TitleConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -35,15 +73,54 @@ class TitleConfig:
 
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'title', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "title", "loglevel", default="info"
+            )
+        )
 
     @property
     def hostname(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'title', 'hostname', default="localhost")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "hostname", default="localhost"
+        )
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'title', 'port', default=8080)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "port", default=8080
+        )
+        
+    @property
+    def port_ssl(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "port_ssl", default=0
+        )
+
+    @property
+    def ssl_key(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "ssl_key", default="cert/title.key"
+        )
+
+    @property
+    def ssl_cert(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "ssl_cert", default="cert/title.pem"
+        )
+
+    @property
+    def reboot_start_time(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "reboot_start_time", default=""
+        )
+
+    @property
+    def reboot_end_time(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "reboot_end_time", default=""
+        )
+
 
 class DatabaseConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -51,43 +128,76 @@ class DatabaseConfig:
 
     @property
     def host(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'host', default="localhost")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "host", default="localhost"
+        )
 
     @property
     def username(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'username', default='aime')
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "username", default="aime"
+        )
 
     @property
     def password(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'password', default='aime')
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "password", default="aime"
+        )
 
     @property
     def name(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'name', default='aime')
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "name", default="aime"
+        )
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'port', default=3306)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "port", default=3306
+        )
 
     @property
     def protocol(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'type', default="mysql")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "type", default="mysql"
+        )
 
     @property
     def sha2_password(self) -> bool:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'sha2_password', default=False)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "sha2_password", default=False
+        )
 
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'database', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "database", "loglevel", default="info"
+            )
+        )
 
     @property
     def user_table_autoincrement_start(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'user_table_autoincrement_start', default=10000)
+        return CoreConfig.get_config_field(
+            self.__config,
+            "core",
+            "database",
+            "user_table_autoincrement_start",
+            default=10000,
+        )
+
+    @property
+    def enable_memcached(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "enable_memcached", default=True
+        )
 
     @property
     def memcached_host(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'memcached_host', default="localhost")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "memcached_host", default="localhost"
+        )
+
 
 class FrontendConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -95,15 +205,24 @@ class FrontendConfig:
 
     @property
     def enable(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'frontend', 'enable', default=False)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "frontend", "enable", default=False
+        )
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'frontend', 'port', default=8090)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "frontend", "port", default=8090
+        )
 
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'frontend', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "frontend", "loglevel", default="info"
+            )
+        )
+
 
 class AllnetConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -111,15 +230,36 @@ class AllnetConfig:
 
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'allnet', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "allnet", "loglevel", default="info"
+            )
+        )
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'allnet', 'port', default=80)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "port", default=80
+        )
+
+    @property
+    def ip_check(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "ip_check", default=False
+        )
 
     @property
     def allow_online_updates(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'allnet', 'allow_online_updates', default=False)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "allow_online_updates", default=False
+        )
+
+    @property
+    def update_cfg_folder(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "update_cfg_folder", default=""
+        )
+
 
 class BillingConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -127,19 +267,28 @@ class BillingConfig:
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'billing', 'port', default=8443)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "billing", "port", default=8443
+        )
 
     @property
     def ssl_key(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'billing', 'ssl_key', default="cert/server.key")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "billing", "ssl_key", default="cert/server.key"
+        )
 
     @property
     def ssl_cert(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'billing', 'ssl_cert', default="cert/server.pem")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "billing", "ssl_cert", default="cert/server.pem"
+        )
 
     @property
     def signing_key(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'billing', 'signing_key', default="cert/billing.key")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "billing", "signing_key", default="cert/billing.key"
+        )
+
 
 class AimedbConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -147,15 +296,36 @@ class AimedbConfig:
 
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'aimedb', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "aimedb", "loglevel", default="info"
+            )
+        )
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'aimedb', 'port', default=22345)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "port", default=22345
+        )
 
     @property
     def key(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'aimedb', 'key', default="")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "key", default=""
+        )
+
+    @property
+    def id_secret(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "id_secret", default=""
+        )
+
+    @property
+    def id_lifetime_seconds(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "id_lifetime_seconds", default=86400
+        )
+
 
 class MuchaConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -163,27 +333,24 @@ class MuchaConfig:
 
     @property
     def enable(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'enable', default=False)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "mucha", "enable", default=False
+        )
 
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "mucha", "loglevel", default="info"
+            )
+        )
 
     @property
     def hostname(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'hostname', default="localhost")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "mucha", "hostname", default="localhost"
+        )
 
-    @property
-    def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'port', default=8444)
-    
-    @property
-    def ssl_cert(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'ssl_cert', default="cert/server.pem")
-    
-    @property
-    def signing_key(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'signing_key', default="cert/billing.key")
 
 class CoreConfig(dict):
     def __init__(self) -> None:
@@ -194,25 +361,28 @@ class CoreConfig(dict):
         self.allnet = AllnetConfig(self)
         self.billing = BillingConfig(self)
         self.aimedb = AimedbConfig(self)
+        self.mucha = MuchaConfig(self)
 
     @classmethod
     def str_to_loglevel(cls, level_str: str):
         if level_str.lower() == "error":
             return logging.ERROR
-        elif level_str.lower().startswith("warn"): # Fits warn or warning
+        elif level_str.lower().startswith("warn"):  # Fits warn or warning
             return logging.WARN
         elif level_str.lower() == "debug":
             return logging.DEBUG
         else:
-             return logging.INFO
+            return logging.INFO
 
     @classmethod
-    def get_config_field(cls, __config: dict, module, *path: str, default: Any = "") -> Any:
+    def get_config_field(
-        envKey = f'CFG_{module}_'
+        cls, __config: dict, module, *path: str, default: Any = ""
+    ) -> Any:
+        envKey = f"CFG_{module}_"
         for arg in path:
-            envKey += arg + '_'
+            envKey += arg + "_"
 
-        if envKey.endswith('_'):
+        if envKey.endswith("_"):
             envKey = envKey[:-1]
 
         if envKey in os.environ:

core/const.py

@@ -1,6 +1,7 @@
 from enum import Enum
 
-class MainboardPlatformCodes():
+
+class MainboardPlatformCodes:
     RINGEDGE = "AALE"
     RINGWIDE = "AAML"
     NU = "AAVE"
@@ -8,7 +9,8 @@ class MainboardPlatformCodes():
     ALLS_UX = "ACAE"
     ALLS_HX = "ACAX"
 
-class MainboardRevisions():
+
+class MainboardRevisions:
     RINGEDGE = 1
     RINGEDGE2 = 2
 
@@ -26,11 +28,70 @@ class MainboardRevisions():
     ALLS_UX2 = 2
     ALLS_HX2 = 12
 
-class KeychipPlatformsCodes():
+
+class KeychipPlatformsCodes:
     RING = "A72E"
     NU = ("A60E", "A60E", "A60E")
     NUSX = ("A61X", "A69X")
     ALLS = "A63E"
 
-class RegionIDs(Enum):
+
-    pass
+class AllnetCountryCode(Enum):
+    JAPAN = "JPN"
+    UNITED_STATES = "USA"
+    HONG_KONG = "HKG"
+    SINGAPORE = "SGP"
+    SOUTH_KOREA = "KOR"
+    TAIWAN = "TWN"
+    CHINA = "CHN"
+
+
+class AllnetJapanRegionId(Enum):
+    NONE = 0
+    AICHI = 1
+    AOMORI = 2
+    AKITA = 3
+    ISHIKAWA = 4
+    IBARAKI = 5
+    IWATE = 6
+    EHIME = 7
+    OITA = 8
+    OSAKA = 9
+    OKAYAMA = 10
+    OKINAWA = 11
+    KAGAWA = 12
+    KAGOSHIMA = 13
+    KANAGAWA = 14
+    GIFU = 15
+    KYOTO = 16
+    KUMAMOTO = 17
+    GUNMA = 18
+    KOCHI = 19
+    SAITAMA = 20
+    SAGA = 21
+    SHIGA = 22
+    SHIZUOKA = 23
+    SHIMANE = 24
+    CHIBA = 25
+    TOKYO = 26
+    TOKUSHIMA = 27
+    TOCHIGI = 28
+    TOTTORI = 29
+    TOYAMA = 30
+    NAGASAKI = 31
+    NAGANO = 32
+    NARA = 33
+    NIIGATA = 34
+    HYOGO = 35
+    HIROSHIMA = 36
+    FUKUI = 37
+    FUKUOKA = 38
+    FUKUSHIMA = 39
+    HOKKAIDO = 40
+    MIE = 41
+    MIYAGI = 42
+    MIYAZAKI = 43
+    YAMAGATA = 44
+    YAMAGUCHI = 45
+    YAMANASHI = 46
+    WAKAYAMA = 47

core/data/cache.py

@@ -1,4 +1,3 @@
-
 from typing import Any, Callable
 from functools import wraps
 import hashlib
@@ -6,17 +5,19 @@ import pickle
 import logging
 from core.config import CoreConfig
 
-cfg:CoreConfig = None # type: ignore
+cfg: CoreConfig = None  # type: ignore
 # Make memcache optional
 try:
     import pylibmc  # type: ignore
+
     has_mc = True
 except ModuleNotFoundError:
     has_mc = False
 
-def cached(lifetime: int=10, extra_key: Any=None) -> Callable:
+
+def cached(lifetime: int = 10, extra_key: Any = None) -> Callable:
     def _cached(func: Callable) -> Callable:
-        if has_mc:
+        if has_mc and (cfg and cfg.database.enable_memcached):
             hostname = "127.0.0.1"
             if cfg:
                 hostname = cfg.database.memcached_host
@@ -26,7 +27,6 @@ def cached(lifetime: int=10, extra_key: Any=None) -> Callable:
             @wraps(func)
             def wrapper(*args: Any, **kwargs: Any) -> Any:
                 if lifetime is not None:
-
                     # Hash function args
                     items = kwargs.items()
                     hashable_args = (args[1:], sorted(list(items)))
@@ -55,7 +55,9 @@ def cached(lifetime: int=10, extra_key: Any=None) -> Callable:
                     memcache.set(cache_key, result, lifetime)
 
                 return result
+
         else:
+
             @wraps(func)
             def wrapper(*args: Any, **kwargs: Any) -> Any:
                 return func(*args, **kwargs)

core/data/database.py

@@ -1,43 +1,68 @@
 import logging, coloredlogs
-from typing import Any, Dict, List
+from typing import Optional, Dict, List
 from sqlalchemy.orm import scoped_session, sessionmaker
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy import create_engine
 from logging.handlers import TimedRotatingFileHandler
-
+import importlib, os
+import secrets, string
+import bcrypt
 from hashlib import sha256
 
 from core.config import CoreConfig
 from core.data.schema import *
+from core.utils import Utils
+
 
 class Data:
+    current_schema_version = 6
+    engine = None
+    session = None
+    user = None
+    arcade = None
+    card = None
+    base = None
     def __init__(self, cfg: CoreConfig) -> None:
         self.config = cfg
 
         if self.config.database.sha2_password:
             passwd = sha256(self.config.database.password.encode()).digest()
-            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
+            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4"
         else:
-            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
+            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4"
 
-        self.__engine = create_engine(self.__url, pool_recycle=3600)
+        if Data.engine is None:
-        session = sessionmaker(bind=self.__engine, autoflush=True, autocommit=True)
+            Data.engine = create_engine(self.__url, pool_recycle=3600)
-        self.session = scoped_session(session)
+            self.__engine = Data.engine
 
-        self.user = UserData(self.config, self.session)
+        if Data.session is None:
-        self.arcade = ArcadeData(self.config, self.session)
+            s = sessionmaker(bind=Data.engine, autoflush=True, autocommit=True)
-        self.card = CardData(self.config, self.session)
+            Data.session = scoped_session(s)
-        self.base = BaseData(self.config, self.session)
+
-        self.schema_ver_latest = 1
+        if Data.user is None:
+            Data.user = UserData(self.config, self.session)
+        
+        if Data.arcade is None:
+            Data.arcade = ArcadeData(self.config, self.session)
+        
+        if Data.card is None:
+            Data.card = CardData(self.config, self.session)
+        
+        if Data.base is None:
+            Data.base = BaseData(self.config, self.session)
 
-        log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
         self.logger = logging.getLogger("database")
 
         # Prevent the logger from adding handlers multiple times
-        if not getattr(self.logger, 'handler_set', None):
+        if not getattr(self.logger, "handler_set", None):            
-            fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.log_dir, "db"), encoding="utf-8",
+            log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
-                when="d", backupCount=10)
+            log_fmt = logging.Formatter(log_fmt_str)
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.config.server.log_dir, "db"),
+                encoding="utf-8",
+                when="d",
+                backupCount=10,
+            )
             fileHandler.setFormatter(log_fmt)
 
             consoleHandler = logging.StreamHandler()
@@ -47,7 +72,286 @@ class Data:
             self.logger.addHandler(consoleHandler)
 
             self.logger.setLevel(self.config.database.loglevel)
-            coloredlogs.install(cfg.database.loglevel, logger=self.logger, fmt=log_fmt_str)
+            coloredlogs.install(
-            self.logger.handler_set = True # type: ignore
+                cfg.database.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.handler_set = True  # type: ignore
 
+    def create_database(self):
+        self.logger.info("Creating databases...")
+        try:
+            metadata.create_all(self.__engine.connect())
+        except SQLAlchemyError as e:
+            self.logger.error(f"Failed to create databases! {e}")
+            return
 
+        games = Utils.get_all_titles()
+        for game_dir, game_mod in games.items():
+            try:
+                if hasattr(game_mod, "database") and hasattr(
+                    game_mod, "current_schema_version"
+                ):
+                    game_mod.database(self.config)
+                metadata.create_all(self.__engine.connect())
+
+                self.base.touch_schema_ver(
+                    game_mod.current_schema_version, game_mod.game_codes[0]
+                )
+
+            except Exception as e:
+                self.logger.warning(
+                    f"Could not load database schema from {game_dir} - {e}"
+                )
+
+        self.logger.info(f"Setting base_schema_ver to {self.current_schema_version}")
+        self.base.set_schema_ver(self.current_schema_version)
+
+        self.logger.info(
+            f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
+        )
+        self.user.reset_autoincrement(
+            self.config.database.user_table_autoincrement_start
+        )
+
+    def recreate_database(self):
+        self.logger.info("Dropping all databases...")
+        self.base.execute("SET FOREIGN_KEY_CHECKS=0")
+        try:
+            metadata.drop_all(self.__engine.connect())
+        except SQLAlchemyError as e:
+            self.logger.error(f"Failed to drop databases! {e}")
+            return
+
+        for root, dirs, files in os.walk("./titles"):
+            for dir in dirs:
+                if not dir.startswith("__"):
+                    try:
+                        mod = importlib.import_module(f"titles.{dir}")
+
+                        try:
+                            if hasattr(mod, "database"):
+                                mod.database(self.config)
+                            metadata.drop_all(self.__engine.connect())
+
+                        except Exception as e:
+                            self.logger.warning(
+                                f"Could not load database schema from {dir} - {e}"
+                            )
+
+                    except ImportError as e:
+                        self.logger.warning(
+                            f"Failed to load database schema dir {dir} - {e}"
+                        )
+            break
+
+        self.base.execute("SET FOREIGN_KEY_CHECKS=1")
+
+        self.create_database()
+
+    def migrate_database(self, game: str, version: Optional[int], action: str) -> None:
+        old_ver = self.base.get_schema_ver(game)
+        sql = ""
+        if version is None:
+            if not game == "CORE":
+                titles = Utils.get_all_titles()
+
+                for folder, mod in titles.items():
+                    if not mod.game_codes[0] == game:
+                        continue
+
+                    if hasattr(mod, "current_schema_version"):
+                        version = mod.current_schema_version
+
+                    else:
+                        self.logger.warning(
+                            f"current_schema_version not found for {folder}"
+                        )
+
+            else:
+                version = self.current_schema_version
+
+        if version is None:
+            self.logger.warning(
+                f"Could not determine latest version for {game}, please specify --version"
+            )
+
+        if old_ver is None:
+            self.logger.error(
+                f"Schema for game {game} does not exist, did you run the creation script?"
+            )
+            return
+
+        if old_ver == version:
+            self.logger.info(
+                f"Schema for game {game} is already version {old_ver}, nothing to do"
+            )
+            return
+
+        if action == "upgrade":
+            for x in range(old_ver, version):
+                if not os.path.exists(
+                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql"
+                ):
+                    self.logger.error(
+                        f"Could not find {action} script {game.upper()}_{x + 1}_{action}.sql in core/data/schema/versions folder"
+                    )
+                    return
+
+                with open(
+                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql",
+                    "r",
+                    encoding="utf-8",
+                ) as f:
+                    sql = f.read()
+
+                result = self.base.execute(sql)
+                if result is None:
+                    self.logger.error("Error execuing sql script!")
+                    return None
+
+        else:
+            for x in range(old_ver, version, -1):
+                if not os.path.exists(
+                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql"
+                ):
+                    self.logger.error(
+                        f"Could not find {action} script {game.upper()}_{x - 1}_{action}.sql in core/data/schema/versions folder"
+                    )
+                    return
+
+                with open(
+                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql",
+                    "r",
+                    encoding="utf-8",
+                ) as f:
+                    sql = f.read()
+
+                result = self.base.execute(sql)
+                if result is None:
+                    self.logger.error("Error execuing sql script!")
+                    return None
+
+        result = self.base.set_schema_ver(version, game)
+        if result is None:
+            self.logger.error("Error setting version in schema_version table!")
+            return None
+
+        self.logger.info(f"Successfully migrated {game} to schema version {version}")
+
+    def create_owner(self, email: Optional[str] = None) -> None:
+        pw = "".join(
+            secrets.choice(string.ascii_letters + string.digits) for i in range(20)
+        )
+        hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt())
+
+        user_id = self.user.create_user(email=email, permission=255, password=hash)
+        if user_id is None:
+            self.logger.error(f"Failed to create owner with email {email}")
+            return
+
+        card_id = self.card.create_card(user_id, "00000000000000000000")
+        if card_id is None:
+            self.logger.error(f"Failed to create card for owner with id {user_id}")
+            return
+
+        self.logger.warning(
+            f"Successfully created owner with email {email}, access code 00000000000000000000, and password {pw} Make sure to change this password and assign a real card ASAP!"
+        )
+
+    def migrate_card(self, old_ac: str, new_ac: str, should_force: bool) -> None:
+        if old_ac == new_ac:
+            self.logger.error("Both access codes are the same!")
+            return
+
+        new_card = self.card.get_card_by_access_code(new_ac)
+        if new_card is None:
+            self.card.update_access_code(old_ac, new_ac)
+            return
+
+        if not should_force:
+            self.logger.warning(
+                f"Card already exists for access code {new_ac} (id {new_card['id']}). If you wish to continue, rerun with the '--force' flag."
+                f" All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
+            )
+            return
+
+        self.logger.info(
+            f"All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
+        )
+        self.card.delete_card(new_card["id"])
+        self.card.update_access_code(old_ac, new_ac)
+
+        hanging_user = self.user.get_user(new_card["user"])
+        if hanging_user["password"] is None:
+            self.logger.info(f"Delete hanging user {hanging_user['id']}")
+            self.user.delete_user(hanging_user["id"])
+
+    def delete_hanging_users(self) -> None:
+        """
+        Finds and deletes users that have not registered for the webui that have no cards assocated with them.
+        """
+        unreg_users = self.user.get_unregistered_users()
+        if unreg_users is None:
+            self.logger.error("Error occoured finding unregistered users")
+
+        for user in unreg_users:
+            cards = self.card.get_user_cards(user["id"])
+            if cards is None:
+                self.logger.error(f"Error getting cards for user {user['id']}")
+                continue
+
+            if not cards:
+                self.logger.info(f"Delete hanging user {user['id']}")
+                self.user.delete_user(user["id"])
+
+    def autoupgrade(self) -> None:
+        all_game_versions = self.base.get_all_schema_vers()
+        if all_game_versions is None:
+            self.logger.warning("Failed to get schema versions")
+            return
+
+        all_games = Utils.get_all_titles()
+        all_games_list: Dict[str, int] = {}
+        for _, mod in all_games.items():
+            if hasattr(mod, "current_schema_version"):
+                all_games_list[mod.game_codes[0]] = mod.current_schema_version
+
+        for x in all_game_versions:
+            failed = False
+            game = x["game"].upper()
+            update_ver = int(x["version"])
+            latest_ver = all_games_list.get(game, 1)
+            if game == "CORE":
+                latest_ver = self.current_schema_version
+
+            if update_ver == latest_ver:
+                self.logger.info(f"{game} is already latest version")
+                continue
+
+            for y in range(update_ver + 1, latest_ver + 1):
+                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
+                    with open(
+                        f"core/data/schema/versions/{game}_{y}_upgrade.sql",
+                        "r",
+                        encoding="utf-8",
+                    ) as f:
+                        sql = f.read()
+
+                    result = self.base.execute(sql)
+                    if result is None:
+                        self.logger.error(
+                            f"Error execuing sql script for game {game} v{y}!"
+                        )
+                        failed = True
+                        break
+                else:
+                    self.logger.warning(f"Could not find script {game}_{y}_upgrade.sql")
+                    failed = True
+
+            if not failed:
+                self.base.set_schema_ver(latest_ver, game)
+    
+    def show_versions(self) -> None:
+        all_game_versions = self.base.get_all_schema_vers()
+        for ver in all_game_versions:
+            self.logger.info(f"{ver['game']} -> v{ver['version']}")

core/data/schema/arcade.py

@@ -1,11 +1,14 @@
-from typing import Optional, Dict
+from typing import Optional, Dict, List
-from sqlalchemy import Table, Column
+from sqlalchemy import Table, Column, and_, or_
 from sqlalchemy.sql.schema import ForeignKey, PrimaryKeyConstraint
-from sqlalchemy.types import Integer, String, Boolean
+from sqlalchemy.types import Integer, String, Boolean, JSON
 from sqlalchemy.sql import func, select
 from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.engine import Row
+import re
 
 from core.data.schema.base import BaseData, metadata
+from core.const import *
 
 arcade = Table(
     "arcade",
@@ -19,95 +22,211 @@ arcade = Table(
     Column("city", String(255)),
     Column("region_id", Integer),
     Column("timezone", String(255)),
-    mysql_charset='utf8mb4'
+    Column("ip", String(39)),
+    mysql_charset="utf8mb4",
 )
 
 machine = Table(
     "machine",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
-    Column("arcade", ForeignKey("arcade.id", ondelete="cascade", onupdate="cascade"), nullable=False),    
+    Column(
+        "arcade",
+        ForeignKey("arcade.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
     Column("serial", String(15), nullable=False),
     Column("board", String(15)),
     Column("game", String(4)),
-    Column("country", String(3)), # overwrites if not null
+    Column("country", String(3)),  # overwrites if not null
     Column("timezone", String(255)),
     Column("ota_enable", Boolean),
+    Column("memo", String(255)),
     Column("is_cab", Boolean),
-    mysql_charset='utf8mb4'
+    Column("data", JSON),
+    mysql_charset="utf8mb4",
 )
 
 arcade_owner = Table(
-    'arcade_owner',
+    "arcade_owner",
     metadata,
-    Column('user', Integer, ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column(
-    Column('arcade', Integer, ForeignKey("arcade.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+        "user",
-    Column('permissions', Integer, nullable=False),
+        Integer,
-    PrimaryKeyConstraint('user', 'arcade', name='arcade_owner_pk'),
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
-    mysql_charset='utf8mb4'
+        nullable=False,
+    ),
+    Column(
+        "arcade",
+        Integer,
+        ForeignKey("arcade.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("permissions", Integer, nullable=False),
+    PrimaryKeyConstraint("user", "arcade", name="arcade_owner_pk"),
+    mysql_charset="utf8mb4",
 )
 
+
 class ArcadeData(BaseData):
-    def get_machine(self, serial: str = None, id: int = None) -> Optional[Dict]:
+    def get_machine(self, serial: str = None, id: int = None) -> Optional[Row]:
         if serial is not None:
-            sql = machine.select(machine.c.serial == serial)
+            serial = serial.replace("-", "")
+            if len(serial) == 11:
+                sql = machine.select(machine.c.serial.like(f"{serial}%"))
+
+            elif len(serial) == 15:
+                sql = machine.select(machine.c.serial == serial)
+
+            else:
+                self.logger.error(f"{__name__ }: Malformed serial {serial}")
+                return None
+
         elif id is not None:
             sql = machine.select(machine.c.id == id)
+
         else:
             self.logger.error(f"{__name__ }: Need either serial or ID to look up!")
             return None
 
         result = self.execute(sql)
-        if result is None: return None
+        if result is None:
+            return None
         return result.fetchone()
 
-    def put_machine(self, arcade_id: int, serial: str = None, board: str = None, game: str = None, is_cab: bool = False) -> Optional[int]:
+    def put_machine(
+        self,
+        arcade_id: int,
+        serial: str = "",
+        board: str = None,
+        game: str = None,
+        is_cab: bool = False,
+    ) -> Optional[int]:
         if arcade_id:
             self.logger.error(f"{__name__ }: Need arcade id!")
             return None
 
-        if serial is None:
+        sql = machine.insert().values(
-            pass
+            arcade=arcade_id, keychip=serial, board=board, game=game, is_cab=is_cab
-
-        sql = machine.insert().values(arcade = arcade_id, keychip = serial, board = board, game = game, is_cab = is_cab)
-        
-        result = self.execute(sql)
-        if result is None: return None
-        return result.lastrowid
-    
-    def get_arcade(self, id: int) -> Optional[Dict]:
-        sql = arcade.select(arcade.c.id == id)
-        result = self.execute(sql)
-        if result is None: return None
-        return result.fetchone()
-    
-    def put_arcade(self, name: str, nickname: str = None, country: str = "JPN", country_id: int = 1, 
-        state: str = "", city: str = "", regional_id: int = 1) -> Optional[int]:
-        if nickname is None: nickname = name
-
-        sql = arcade.insert().values(name = name, nickname = nickname, country = country, country_id = country_id,
-        state = state, city = city, regional_id = regional_id)
-
-        result = self.execute(sql)
-        if result is None: return None
-        return result.lastrowid
-
-    def get_arcade_owners(self, arcade_id: int) -> Optional[Dict]:
-        sql = select(arcade_owner).where(arcade_owner.c.arcade==arcade_id)
-
-        result = self.execute(sql)
-        if result is None: return None
-        return result.fetchall()
-
-    def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
-        sql = insert(arcade_owner).values(
-            arcade=arcade_id,
-            user=user_id
         )
 
         result = self.execute(sql)
-        if result is None: return None
+        if result is None:
+            return None
         return result.lastrowid
 
-    def generate_keychip_serial(self, platform_id: int) -> str:
+    def set_machine_serial(self, machine_id: int, serial: str) -> None:
-        pass
+        result = self.execute(
+            machine.update(machine.c.id == machine_id).values(keychip=serial)
+        )
+        if result is None:
+            self.logger.error(
+                f"Failed to update serial for machine {machine_id} -> {serial}"
+            )
+        return result.lastrowid
+
+    def set_machine_boardid(self, machine_id: int, boardid: str) -> None:
+        result = self.execute(
+            machine.update(machine.c.id == machine_id).values(board=boardid)
+        )
+        if result is None:
+            self.logger.error(
+                f"Failed to update board id for machine {machine_id} -> {boardid}"
+            )
+
+    def get_arcade(self, id: int) -> Optional[Row]:
+        sql = arcade.select(arcade.c.id == id)
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+    
+    def get_arcade_machines(self, id: int) -> Optional[List[Row]]:
+        sql = machine.select(machine.c.arcade == id)
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def put_arcade(
+        self,
+        name: str,
+        nickname: str = None,
+        country: str = "JPN",
+        country_id: int = 1,
+        state: str = "",
+        city: str = "",
+        regional_id: int = 1,
+    ) -> Optional[int]:
+        if nickname is None:
+            nickname = name
+
+        sql = arcade.insert().values(
+            name=name,
+            nickname=nickname,
+            country=country,
+            country_id=country_id,
+            state=state,
+            city=city,
+            regional_id=regional_id,
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def get_arcades_managed_by_user(self, user_id: int) -> Optional[List[Row]]:
+        sql = select(arcade).join(arcade_owner, arcade_owner.c.arcade == arcade.c.id).where(arcade_owner.c.user == user_id)
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchall()
+    
+    def get_manager_permissions(self, user_id: int, arcade_id: int) -> Optional[int]:
+        sql = select(arcade_owner.c.permissions).where(and_(arcade_owner.c.user == user_id, arcade_owner.c.arcade == arcade_id))
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchone()
+
+    def get_arcade_owners(self, arcade_id: int) -> Optional[Row]:
+        sql = select(arcade_owner).where(arcade_owner.c.arcade == arcade_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
+        sql = insert(arcade_owner).values(arcade=arcade_id, user=user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def format_serial(
+        self, platform_code: str, platform_rev: int, serial_num: int, append: int = 4152
+    ) -> str:
+        return f"{platform_code}{platform_rev:02d}A{serial_num:04d}{append:04d}"  # 0x41 = A, 0x52 = R
+
+    def validate_keychip_format(self, serial: str) -> bool:
+        if re.fullmatch(r"^A[0-9]{2}[E|X][-]?[0-9]{2}[A-HJ-NP-Z][0-9]{4}([0-9]{4})?$", serial) is None:
+            return False
+        
+        return True
+
+    def get_arcade_by_name(self, name: str) -> Optional[List[Row]]:
+        sql = arcade.select(or_(arcade.c.name.like(f"%{name}%"), arcade.c.nickname.like(f"%{name}%")))
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_arcades_by_ip(self, ip: str) -> Optional[List[Row]]:
+        sql = arcade.select().where(arcade.c.ip == ip)
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()

core/data/schema/base.py

@@ -2,6 +2,7 @@ import json
 import logging
 from random import randrange
 from typing import Any, Optional, Dict, List
+from sqlalchemy.engine import Row
 from sqlalchemy.engine.cursor import CursorResult
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.sql import text, func, select
@@ -19,7 +20,7 @@ schema_ver = Table(
     metadata,
     Column("game", String(4), primary_key=True, nullable=False),
     Column("version", Integer, nullable=False, server_default="1"),
-    mysql_charset='utf8mb4'
+    mysql_charset="utf8mb4",
 )
 
 event_log = Table(
@@ -29,22 +30,24 @@ event_log = Table(
     Column("system", String(255), nullable=False),
     Column("type", String(255), nullable=False),
     Column("severity", Integer, nullable=False),
+    Column("message", String(1000), nullable=False),
     Column("details", JSON, nullable=False),
     Column("when_logged", TIMESTAMP, nullable=False, server_default=func.now()),
-    mysql_charset='utf8mb4'
+    mysql_charset="utf8mb4",
 )
 
-class BaseData():
+
+class BaseData:
     def __init__(self, cfg: CoreConfig, conn: Connection) -> None:
         self.config = cfg
         self.conn = conn
         self.logger = logging.getLogger("database")
 
-    def execute(self, sql: str, opts: Dict[str, Any]={}) -> Optional[CursorResult]:
+    def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
         res = None
 
         try:
-            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())} || {opts}")
+            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())}")
             res = self.conn.execute(text(sql), opts)
 
         except SQLAlchemyError as e:
@@ -55,7 +58,7 @@ class BaseData():
             self.logger.error(f"UnicodeEncodeError error {e}")
             return None
 
-        except:
+        except Exception:
             try:
                 res = self.conn.execute(sql, opts)
 
@@ -67,7 +70,7 @@ class BaseData():
                 self.logger.error(f"UnicodeEncodeError error {e}")
                 return None
 
-            except:
+            except Exception:
                 self.logger.error(f"Unknown error")
                 raise
 
@@ -79,30 +82,67 @@ class BaseData():
         """
         return randrange(10000, 9999999)
 
+    def get_all_schema_vers(self) -> Optional[List[Row]]:
+        sql = select(schema_ver)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def get_schema_ver(self, game: str) -> Optional[int]:
         sql = select(schema_ver).where(schema_ver.c.game == game)
 
         result = self.execute(sql)
         if result is None:
             return None
-        return result.fetchone()["version"]
 
-    def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
+        row = result.fetchone()
-        sql = insert(schema_ver).values(game = game, version = ver)
+        if row is None:
-        conflict = sql.on_duplicate_key_update(version = ver)
+            return None
+
+        return row["version"]
+    
+    def touch_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
+        sql = insert(schema_ver).values(game=game, version=ver)
+        conflict = sql.on_duplicate_key_update(version=schema_ver.c.version)
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.error(f"Failed to update schema version for game {game} (v{ver})")
+            self.logger.error(
+                f"Failed to update schema version for game {game} (v{ver})"
+            )
             return None
         return result.lastrowid
 
-    def log_event(self, system: str, type: str, severity: int, details: Dict) -> Optional[int]:
+    def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = event_log.insert().values(system = system, type = type, severity = severity, details = json.dumps(details))
+        sql = insert(schema_ver).values(game=game, version=ver)
+        conflict = sql.on_duplicate_key_update(version=ver)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(
+                f"Failed to update schema version for game {game} (v{ver})"
+            )
+            return None
+        return result.lastrowid
+
+    def log_event(
+        self, system: str, type: str, severity: int, message: str, details: Dict = {}
+    ) -> Optional[int]:
+        sql = event_log.insert().values(
+            system=system,
+            type=type,
+            severity=severity,
+            message=message,
+            details=json.dumps(details),
+        )
         result = self.execute(sql)
 
         if result is None:
-            self.logger.error(f"{__name__}: Failed to insert event into event log! system = {system}, type = {type}, severity = {severity}, details = {details}")
+            self.logger.error(
+                f"{__name__}: Failed to insert event into event log! system = {system}, type = {type}, severity = {severity}, message = {message}"
+            )
             return None
 
         return result.lastrowid
@@ -111,11 +151,14 @@ class BaseData():
         sql = event_log.select().limit(entries).all()
         result = self.execute(sql)
 
-        if result is None: return None
+        if result is None:
+            return None
         return result.fetchall()
 
     def fix_bools(self, data: Dict) -> Dict:
-        for k,v in data.items():
+        for k, v in data.items():
+            if k == "userName" or k == "teamName":
+                continue
             if type(v) == str and v.lower() == "true":
                 data[k] = True
             elif type(v) == str and v.lower() == "false":

core/data/schema/card.py

@@ -3,55 +3,113 @@ from sqlalchemy import Table, Column, UniqueConstraint
 from sqlalchemy.types import Integer, String, Boolean, TIMESTAMP
 from sqlalchemy.sql.schema import ForeignKey
 from sqlalchemy.sql import func
+from sqlalchemy.engine import Row
 
 from core.data.schema.base import BaseData, metadata
 
 aime_card = Table(
-    'aime_card',
+    "aime_card",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
-    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
     Column("access_code", String(20)),
     Column("created_date", TIMESTAMP, server_default=func.now()),
     Column("last_login_date", TIMESTAMP, onupdate=func.now()),
     Column("is_locked", Boolean, server_default="0"),
     Column("is_banned", Boolean, server_default="0"),
     UniqueConstraint("user", "access_code", name="aime_card_uk"),
-    mysql_charset='utf8mb4'
+    mysql_charset="utf8mb4",
 )
 
+
 class CardData(BaseData):
+    def get_card_by_access_code(self, access_code: str) -> Optional[Row]:
+        sql = aime_card.select(aime_card.c.access_code == access_code)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_card_by_id(self, card_id: int) -> Optional[Row]:
+        sql = aime_card.select(aime_card.c.id == card_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def update_access_code(self, old_ac: str, new_ac: str) -> None:
+        sql = aime_card.update(aime_card.c.access_code == old_ac).values(
+            access_code=new_ac
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"Failed to change card access code from {old_ac} to {new_ac}"
+            )
+
     def get_user_id_from_card(self, access_code: str) -> Optional[int]:
         """
         Given a 20 digit access code as a string, get the user id associated with that card
         """
-        sql = aime_card.select(aime_card.c.access_code == access_code)
+        card = self.get_card_by_access_code(access_code)
-
+        if card is None:
-        result = self.execute(sql)
+            return None
-        if result is None: return None 
-
-        card = result.fetchone()
-        if card is None: return None
 
         return int(card["user"])
 
-    def get_user_cards(self, aime_id: int) -> Optional[List[Dict]]:
+    def get_card_banned(self, access_code: str) -> Optional[bool]:
+        """
+        Given a 20 digit access code as a string, check if the card is banned
+        """
+        card = self.get_card_by_access_code(access_code)
+        if card is None:
+            return None
+        if card["is_banned"]:
+            return True
+        return False
+    def get_card_locked(self, access_code: str) -> Optional[bool]:
+        """
+        Given a 20 digit access code as a string, check if the card is locked
+        """
+        card = self.get_card_by_access_code(access_code)
+        if card is None:
+            return None
+        if card["is_locked"]:
+            return True
+        return False
+
+    def delete_card(self, card_id: int) -> None:
+        sql = aime_card.delete(aime_card.c.id == card_id)
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(f"Failed to delete card with id {card_id}")
+
+    def get_user_cards(self, aime_id: int) -> Optional[List[Row]]:
         """
         Returns all cards owned by a user
         """
         sql = aime_card.select(aime_card.c.user == aime_id)
         result = self.execute(sql)
-        if result is None: return None        
+        if result is None:
+            return None
         return result.fetchall()
 
-    
     def create_card(self, user_id: int, access_code: str) -> Optional[int]:
         """
         Given a aime_user id and a 20 digit access code as a string, create a card and return the ID if successful
         """
         sql = aime_card.insert().values(user=user_id, access_code=access_code)
         result = self.execute(sql)
-        if result is None: return None
+        if result is None:
+            return None
         return result.lastrowid
 
     def to_access_code(self, luid: str) -> str:
@@ -64,4 +122,4 @@ class CardData(BaseData):
         """
         Given a 20 digit access code as a string, return the 16 hex character luid
         """
-        return f'{int(access_code):0{16}x}'
+        return f"{int(access_code):0{16}x}"

core/data/schema/user.py

@@ -1,9 +1,12 @@
 from enum import Enum
-from typing import Dict, Optional
+from typing import Optional, List
 from sqlalchemy import Table, Column
 from sqlalchemy.types import Integer, String, TIMESTAMP
-from sqlalchemy.sql.schema import ForeignKey
 from sqlalchemy.sql import func
+from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.sql import func, select
+from sqlalchemy.engine import Row
+import bcrypt
 
 from core.data.schema.base import BaseData, metadata
 
@@ -18,40 +21,103 @@ aime_user = Table(
     Column("created_date", TIMESTAMP, server_default=func.now()),
     Column("last_login_date", TIMESTAMP, onupdate=func.now()),
     Column("suspend_expire_time", TIMESTAMP),
-    mysql_charset='utf8mb4'
+    mysql_charset="utf8mb4",
 )
 
-frontend_session = Table(
-    "frontend_session",
-    metadata,
-    Column("id", Integer, primary_key=True, unique=True),
-    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
-    Column('session_cookie', String(32), nullable=False, unique=True),
-    Column("expires", TIMESTAMP, nullable=False),
-    mysql_charset='utf8mb4'
-)
 
 class PermissionBits(Enum):
     PermUser = 1
     PermMod = 2
     PermSysAdmin = 4
 
+
 class UserData(BaseData):
-    def create_user(self, username: str = None, email: str = None, password: str = None) -> Optional[int]:
+    def create_user(
-
+        self,
-        if email is None:
+        id: int = None,
-            permission = None
+        username: str = None,
+        email: str = None,
+        password: str = None,
+        permission: int = 1,
+    ) -> Optional[int]:
+        if id is None:
+            sql = insert(aime_user).values(
+                username=username,
+                email=email,
+                password=password,
+                permissions=permission,
+            )
         else:
-            permission = 0
+            sql = insert(aime_user).values(
+                id=id,
+                username=username,
+                email=email,
+                password=password,
+                permissions=permission,
+            )
 
-        sql = aime_user.insert().values(username=username, email=email, password=password, permissions=permission)
+        conflict = sql.on_duplicate_key_update(
+            username=username, email=email, password=password, permissions=permission
+        )
 
-        result = self.execute(sql)
+        result = self.execute(conflict)
-        if result is None: return None
+        if result is None:
+            return None
         return result.lastrowid
 
+    def get_user(self, user_id: int) -> Optional[Row]:
+        sql = select(aime_user).where(aime_user.c.id == user_id)
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchone()
+
+    def check_password(self, user_id: int, passwd: bytes = None) -> bool:
+        usr = self.get_user(user_id)
+        if usr is None:
+            return False
+
+        if usr["password"] is None:
+            return False
+        
+        if passwd is None or not passwd:
+            return False
+
+        return bcrypt.checkpw(passwd, usr["password"].encode())
+
     def reset_autoincrement(self, ai_value: int) -> None:
-        # Didn't feel like learning how to do this the right way
+        # ALTER TABLE isn't in sqlalchemy so we do this the ugly way
-        # if somebody wants a free PR go nuts I guess
         sql = f"ALTER TABLE aime_user AUTO_INCREMENT={ai_value}"
         self.execute(sql)
+
+    def delete_user(self, user_id: int) -> None:
+        sql = aime_user.delete(aime_user.c.id == user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(f"Failed to delete user with id {user_id}")
+
+    def get_unregistered_users(self) -> List[Row]:
+        """
+        Returns a list of users who have not registered with the webui. They may or may not have cards.
+        """
+        sql = select(aime_user).where(aime_user.c.password == None)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def find_user_by_email(self, email: str) -> Row:
+        sql = select(aime_user).where(aime_user.c.email == email)
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchone()
+
+    def find_user_by_username(self, username: str) -> List[Row]:
+        sql = aime_user.select(aime_user.c.username.like(f"%{username}%"))
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchall()

core/data/schema/versions/CORE_1_rollback.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `frontend_session`
+DROP COLUMN `ip`;

core/data/schema/versions/CORE_2_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE `event_log` DROP COLUMN `message`;

core/data/schema/versions/CORE_2_upgrade.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `frontend_session`
+ADD `ip` CHAR(15);

core/data/schema/versions/CORE_3_rollback.sql

@@ -0,0 +1,12 @@
+CREATE TABLE `frontend_session` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `user` int(11) NOT NULL,
+  `ip` varchar(15) DEFAULT NULL,
+  `session_cookie` varchar(32) NOT NULL,
+  `expires` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `id` (`id`),
+  UNIQUE KEY `session_cookie` (`session_cookie`),
+  KEY `user` (`user`),
+  CONSTRAINT `frontend_session_ibfk_1` FOREIGN KEY (`user`) REFERENCES `aime_user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4;

core/data/schema/versions/CORE_3_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE `event_log` ADD COLUMN `message` VARCHAR(1000) NOT NULL AFTER `severity`;

core/data/schema/versions/CORE_4_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE machine DROP COLUMN memo;
+ALTER TABLE machine DROP COLUMN is_blacklisted;
+ALTER TABLE machine DROP COLUMN `data`;

core/data/schema/versions/CORE_4_upgrade.sql

@@ -0,0 +1 @@
+DROP TABLE `frontend_session`;

core/data/schema/versions/CORE_5_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE arcade DROP COLUMN 'ip';

core/data/schema/versions/CORE_5_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE machine ADD memo varchar(255) NULL;
+ALTER TABLE machine ADD is_blacklisted tinyint(1) NULL;
+ALTER TABLE machine ADD `data` longtext NULL;

core/data/schema/versions/CORE_6_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE arcade ADD ip varchar(39) NULL;

core/data/schema/versions/SBZV_1_rollback.sql

@@ -0,0 +1,9 @@
+SET FOREIGN_KEY_CHECKS=0;
+ALTER TABLE diva_score DROP FOREIGN KEY diva_score_ibfk_1;
+ALTER TABLE diva_score DROP CONSTRAINT diva_score_uk;
+ALTER TABLE diva_score ADD CONSTRAINT diva_score_uk UNIQUE (user, pv_id, difficulty);
+ALTER TABLE diva_score ADD CONSTRAINT diva_score_ibfk_1 FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE;
+
+ALTER TABLE diva_score DROP COLUMN edition;
+ALTER TABLE diva_playlog DROP COLUMN edition;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SBZV_2_rollback.sql

@@ -0,0 +1,17 @@
+ALTER TABLE diva_profile_shop DROP COLUMN c_itm_eqp_ary;
+ALTER TABLE diva_profile_shop DROP COLUMN ms_itm_flg_ary;
+
+ALTER TABLE diva_profile DROP COLUMN use_pv_mdl_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_mdl_pri;
+ALTER TABLE diva_profile DROP COLUMN use_pv_skn_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_btn_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_sld_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_chn_sld_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_sldr_tch_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_mdl_eqp VARCHAR(8) NOT NULL DEFAULT "true" AFTER sort_kind;
+ALTER TABLE diva_profile ADD COLUMN use_pv_btn_se_eqp VARCHAR(8) NOT NULL DEFAULT "true" AFTER use_pv_mdl_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_sld_se_eqp VARCHAR(8) NOT NULL DEFAULT "false" AFTER use_pv_btn_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_chn_sld_se_eqp VARCHAR(8) NOT NULL DEFAULT "false" AFTER use_pv_sld_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_sldr_tch_se_eqp VARCHAR(8) NOT NULL DEFAULT "false" AFTER use_pv_chn_sld_se_eqp;
+
+DROP TABLE IF EXISTS `diva_profile_pv_customize`;

core/data/schema/versions/SBZV_2_upgrade.sql

@@ -0,0 +1,9 @@
+SET FOREIGN_KEY_CHECKS=0;
+ALTER TABLE diva_score ADD COLUMN edition int(11) DEFAULT 0 AFTER difficulty;
+ALTER TABLE diva_playlog ADD COLUMN edition int(11) DEFAULT 0 AFTER difficulty;
+
+ALTER TABLE diva_score DROP FOREIGN KEY diva_score_ibfk_1;
+ALTER TABLE diva_score DROP CONSTRAINT diva_score_uk;
+ALTER TABLE diva_score ADD CONSTRAINT diva_score_uk UNIQUE (user, pv_id, difficulty, edition);
+ALTER TABLE diva_score ADD CONSTRAINT diva_score_ibfk_1 FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SBZV_3_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE diva_profile DROP COLUMN passwd_stat;
+ALTER TABLE diva_profile DROP COLUMN passwd;
+ALTER TABLE diva_profile MODIFY player_name VARCHAR(8);

core/data/schema/versions/SBZV_3_upgrade.sql

@@ -0,0 +1,33 @@
+ALTER TABLE diva_profile_shop ADD COLUMN c_itm_eqp_ary varchar(59) DEFAULT "-999,-999,-999,-999,-999,-999,-999,-999,-999,-999,-999,-999";
+ALTER TABLE diva_profile_shop ADD COLUMN ms_itm_flg_ary varchar(59) DEFAULT "-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1";
+
+ALTER TABLE diva_profile DROP COLUMN use_pv_mdl_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_btn_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_sld_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_chn_sld_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_sldr_tch_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_mdl_eqp BOOLEAN NOT NULL DEFAULT true AFTER sort_kind;
+ALTER TABLE diva_profile ADD COLUMN use_mdl_pri BOOLEAN NOT NULL DEFAULT false AFTER use_pv_mdl_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_skn_eqp BOOLEAN NOT NULL DEFAULT false AFTER use_mdl_pri;
+ALTER TABLE diva_profile ADD COLUMN use_pv_btn_se_eqp BOOLEAN NOT NULL DEFAULT true AFTER use_pv_skn_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_sld_se_eqp BOOLEAN NOT NULL DEFAULT false AFTER use_pv_btn_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_chn_sld_se_eqp BOOLEAN NOT NULL DEFAULT false AFTER use_pv_sld_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_sldr_tch_se_eqp BOOLEAN NOT NULL DEFAULT false AFTER use_pv_chn_sld_se_eqp;
+
+
+CREATE TABLE diva_profile_pv_customize (
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    user INT NOT NULL,
+    version INT NOT NULL,
+    pv_id INT NOT NULL,
+    mdl_eqp_ary VARCHAR(14) DEFAULT '-999,-999,-999',
+    c_itm_eqp_ary VARCHAR(59) DEFAULT '-999,-999,-999,-999,-999,-999,-999,-999,-999,-999,-999,-999',
+    ms_itm_flg_ary VARCHAR(59) DEFAULT '-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1',
+    skin INT DEFAULT '-1',
+    btn_se INT DEFAULT '-1',
+    sld_se INT DEFAULT '-1',
+    chsld_se INT DEFAULT '-1',
+    sldtch_se INT DEFAULT '-1',
+    UNIQUE KEY diva_profile_pv_customize_uk (user, version, pv_id),
+    CONSTRAINT diva_profile_pv_customize_ibfk_1 FOREIGN KEY (user) REFERENCES aime_user (id) ON DELETE CASCADE ON UPDATE CASCADE
+);

core/data/schema/versions/SBZV_4_rollback.sql

@@ -0,0 +1,9 @@
+ALTER TABLE diva_profile
+    DROP cnp_cid,
+    DROP cnp_val,
+    DROP cnp_rr,
+    DROP cnp_sp,
+    DROP btn_se_eqp,
+    DROP sld_se_eqp,
+    DROP chn_sld_se_eqp,
+    DROP sldr_tch_se_eqp;

core/data/schema/versions/SBZV_4_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE diva_profile ADD COLUMN passwd_stat INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE diva_profile ADD COLUMN passwd VARCHAR(12) NOT NULL DEFAULT "**********";
+ALTER TABLE diva_profile MODIFY player_name VARCHAR(10);

core/data/schema/versions/SBZV_5_rollback.sql

@@ -0,0 +1,2 @@
+ALTER TABLE diva_profile
+    DROP skn_eqp;

core/data/schema/versions/SBZV_5_upgrade.sql

@@ -0,0 +1,9 @@
+ALTER TABLE diva_profile
+    ADD cnp_cid INT NOT NULL DEFAULT -1,
+    ADD cnp_val INT NOT NULL DEFAULT -1,
+    ADD cnp_rr INT NOT NULL DEFAULT -1,
+    ADD cnp_sp VARCHAR(255) NOT NULL DEFAULT "",
+    ADD btn_se_eqp INT NOT NULL DEFAULT -1,
+    ADD sld_se_eqp INT NOT NULL DEFAULT -1,
+    ADD chn_sld_se_eqp INT NOT NULL DEFAULT -1,
+    ADD sldr_tch_se_eqp INT NOT NULL DEFAULT -1;

core/data/schema/versions/SBZV_6_upgrade.sql

@@ -0,0 +1,2 @@
+ALTER TABLE diva_profile
+    ADD skn_eqp INT NOT NULL DEFAULT 0;

core/data/schema/versions/SDBT_1_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE chuni_static_music CHANGE COLUMN worldsEndTag worldsEndTag VARCHAR(20) NULL DEFAULT NULL ;

core/data/schema/versions/SDBT_2_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE chuni_score_course DROP COLUMN theoryCount, DROP COLUMN orderId, DROP COLUMN playerRating;

core/data/schema/versions/SDBT_2_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE chuni_static_music CHANGE COLUMN worldsEndTag worldsEndTag VARCHAR(7) NULL DEFAULT NULL ;

core/data/schema/versions/SDBT_3_rollback.sql

@@ -0,0 +1,30 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    DROP COLUMN regionId,
+    DROP COLUMN machineType;
+
+ALTER TABLE chuni_static_events
+    DROP COLUMN startDate;
+
+ALTER TABLE chuni_profile_data
+    DROP COLUMN rankUpChallengeResults;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN presetId id INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD PRIMARY KEY(id);
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_uk UNIQUE(id, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY(presetId)
+    REFERENCES chuni_static_login_bonus_preset(id) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_3_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE chuni_score_course ADD theoryCount int(11), ADD orderId int(11), ADD playerRating int(11);

core/data/schema/versions/SDBT_4_rollback.sql

@@ -0,0 +1,12 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+  CHANGE COLUMN isClear isClear TINYINT(1) NULL DEFAULT NULL;
+
+ALTER TABLE chuni_score_best
+  CHANGE COLUMN isSuccess isSuccess TINYINT(1) NULL DEFAULT NULL ;
+
+ALTER TABLE chuni_score_playlog
+  DROP COLUMN ticketId;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_4_upgrade.sql

@@ -0,0 +1,29 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    ADD COLUMN regionId INT,
+    ADD COLUMN machineType INT;
+
+ALTER TABLE chuni_static_events
+    ADD COLUMN startDate TIMESTAMP NOT NULL DEFAULT current_timestamp();
+
+ALTER TABLE chuni_profile_data
+    ADD COLUMN rankUpChallengeResults JSON;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN id presetId INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP INDEX chuni_static_login_bonus_preset_uk;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_pk PRIMARY KEY (presetId, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY (presetId, version)
+    REFERENCES chuni_static_login_bonus_preset(presetId, version) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_5_upgrade.sql

@@ -0,0 +1,12 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+  CHANGE COLUMN isClear isClear TINYINT(6) NULL DEFAULT NULL;
+
+ALTER TABLE chuni_score_best
+  CHANGE COLUMN isSuccess isSuccess INT(11) NULL DEFAULT NULL ;
+
+ALTER TABLE chuni_score_playlog
+  ADD COLUMN ticketId INT(11) NULL AFTER machineType;
+
+SET FOREIGN_KEY_CHECKS = 1;