Merge branch 'develop' into fork_develop

.gitattributes

@@ -0,0 +1,3 @@
+*.csv binary
+*.txt binary
+*.json binary

Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.9.15-slim-bullseye
 
-RUN apt update && apt install default-libmysqlclient-dev build-essential libtk nodejs npm -y
+RUN apt update && apt install default-libmysqlclient-dev build-essential libtk nodejs npm pkg-config -y
 
 WORKDIR /app
 COPY requirements.txt requirements.txt
@@ -12,10 +12,10 @@ RUN chmod +x entrypoint.sh
 
 COPY index.py index.py
 COPY dbutils.py dbutils.py
+COPY read.py read.py
 ADD core core
 ADD titles titles
-ADD config config
-ADD log log
+ADD logs logs
 ADD cert cert
 
-ENTRYPOINT [ "/app/entrypoint.sh" ]
+ENTRYPOINT [ "/app/entrypoint.sh" ]

LICENSE.txt

@@ -0,0 +1,13 @@
+           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                   Version 2, December 2004
+ 
+Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+Everyone is permitted to copy and distribute verbatim or modified
+copies of this license document, and changing it is allowed as long
+as the name is changed.
+ 
+           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+  TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. You just DO WHAT THE FUCK YOU WANT TO.

changelog.md

@@ -1,6 +1,170 @@
 # Changelog
 Documenting updates to ARTEMiS, to be updated every time the master branch is pushed to.
 
+## 20240302
+### SAO
++ Fixing new profile creation with right heroes and start VP
++ Fix to the Unanalyzed Log responses returning the wrong rewards
++ Documentation revised
+
+## 20240226
+### CXB
++ Fixing paths for rev.py
++ Changed encoding for handle_data_item_list_icon_request
+
+## 20240202
+### SAO
++ Added reader assets and edited the game specific documentation
+
+## 20240118
+### System
++ Added game version names to the readme
+
+## 20240109
+### System
++ Removed `ADD config config` from dockerfile [#83](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/83) (Thanks zaphkito!)
+
+### Aimedb
++ Fixed an error that resulted from trying to scan a banned or locked card
+
+## 20240108
+### System
++ Change how the underlying system handles URLs
+  + This can now allow for things like version-specific, or even keychip-specific URLs
+  + Specific changes to games are noted below
++ Fix docker files [#60](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/60) (Thanks Rylie!)
++ Fix support for python 3.8 - 3.10
+
+### Aimedb
++ Add support for SegaAuth key in games that support it (for now only Chunithm)
+  + This is a JWT that is sent to games, by Aimedb, that the games send to their game server, to verify that the access code the game is sending to the server was obtained via aimedb.
+  + Requires a base64-encoded secret to be set in the `core.yaml`
+
+### Chunithm
++ Fix Air support
++ Add saving for userRecentPlayerList
++ Add support for SegaAuthKey
++ Fix a bug arising if a user set their name to be 'true' or 'false'
++ Add support for Sun+ [#78](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/78) (Thanks EmmyHeart!)
++ Add `matching` section to `chuni.yaml`
++ ~~Change `udpHolePunchUri` and `reflectorUri` to be STUN and TURN servers~~ Reverted
++ Imrpove `GetGameSetting` request handling for different versions
++ Fix issue where songs would not always return all scores [#92](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/92) (Thanks Kumubou!)
+
+### maimai DX
++ Fix user charges failing to save
+
+### maimai
++ Made it functional
+
+### CXB
++ Improvements to request dispatching
++ Add support for non-omnimix music lists
+
+
+### IDZ
++ Fix news urls in accordance with the system change to URLs
+
+### Initial D THE ARCADE
++ Added support for Initial D THE ARCADE S2
+  + Story mode progress added
+  + Bunta Challenge/Touhou Project modes added
+  + Time Trials added
+  + Leaderboards added, but doesn't refresh sometimes
+  + Theory of Street mode added (with CPUs)
+  + Play Stamp/Timetrial events added
+  + Frontend to download profile added
+  + Importer to import profiles added
+
+### ONGEKI
++ Now supports HTTPS on a per-version basis
++ Merg PR [#61](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/61) (Thanks phantomlan!)
+  + Add Ranking Event Support
+  + Add reward list support
+  + Add version segregation to Event Ranking, Tech Challenge, and Music Ranking
+  + Now stores ClientTestmode and ClientSetting data
++ Fix mission points not adding correctly [#68](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/68) (Thanks phantomlan!)
++ Fix tech challenge [#70](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/70) (Thanks phantomlan!)
+
+### SAO
++ Change endpoint in accordance with the system change to URLs
++ Update request header class to be more accurate
++ Encrypted requests are now supported
++ Change to using handler classes instead of raw structs for simplicity
+
+### Wacca
++ Fix a server error causing a seperate error that casued issues
++ Add better error printing
++ Add better request validation
++ Fix HousingStartV2
++ Fix Lily's housing/get handler
+
+## 20231107
+### CXB
++  Hotfix `render_POST` sometimes failing to read the request body on large requests
+
+## 20231106
+### CXB
++ Hotfix `render_POST` function signature signature
++ Hotfix `handle_action_addenergy_request` hard failing if `get_energy` returns None
+
+## 20231015
+### maimai DX
++ Added support for FESTiVAL PLUS
+
+### Card Maker
++ Added support for maimai DX FESTiVAL PLUS
+
+## 20230716
+### General
++ Docker files added (#19)
++ Added support for threading
+  + This comes with the caviat that enabling it will not allow you to use Ctrl + C to stop the server.
+
+### Webui
++ Small improvements
++ Add card display
+
+### Allnet
++ Billing format validation
++ Fix naomitest.html endpoint
++ Add event logging for auths and billing
++ LoaderStateRecorder endpoint handler added
+
+### Mucha
++ Fixed log level always being "Info"
++ Add stub handler for DownloadState
+
+### Sword Art Online
++ Support added
+
+### Crossbeats
++ Added threading to profile loading
+  + This should cause a noticeable speed-up
+
+### Card Maker
++ DX Passes fixed
++ Various improvements
+
+### Diva
++ Added clear status calculation
++ Various minor fixes and improvements
+
+### Maimai
++ Added support for memorial photo uploads
++ Added support for the following versions
+  + Festival
+  + FiNALE
++ Various bug fixes and improvements
+
+### Wacca
++ Fixed an error that sometimes occoured when trying to unlock songs (#22)
+
+### Pokken
++ Profile saving added (loading TBA)
++ Use external STUN server for matching by default
+  + Matching still not working
+
 ## 2023042300
 ### Wacca
 + Time free now works properly

core/__init__.py

@@ -1,6 +1,6 @@
 from core.config import CoreConfig
-from core.allnet import AllnetServlet
-from core.aimedb import AimedbFactory
+from core.allnet import AllnetServlet, BillingServlet
+from core.aimedb import AimedbServlette
 from core.title import TitleServlet
 from core.utils import Utils
 from core.mucha import MuchaServlet

core/adb_handlers/__init__.py

@@ -0,0 +1,6 @@
+from .base import ADBBaseRequest, ADBBaseResponse, ADBHeader, ADBHeaderException, PortalRegStatus, LogStatus, ADBStatus
+from .base import CompanyCodes, ReaderFwVer, CMD_CODE_GOODBYE, HEADER_SIZE
+from .lookup import ADBLookupRequest, ADBLookupResponse, ADBLookupExResponse
+from .campaign import ADBCampaignClearRequest, ADBCampaignClearResponse, ADBCampaignResponse, ADBOldCampaignRequest, ADBOldCampaignResponse
+from .felica import ADBFelicaLookupRequest, ADBFelicaLookupResponse, ADBFelicaLookup2Request, ADBFelicaLookup2Response
+from .log import ADBLogExRequest, ADBLogRequest, ADBStatusLogRequest, ADBLogExResponse

core/adb_handlers/base.py

@@ -0,0 +1,170 @@
+import struct
+from construct import Struct, Int16ul, Int32ul, PaddedString
+from enum import Enum
+import re
+from typing import Union, Final
+
+class LogStatus(Enum):
+    NONE = 0
+    START = 1
+    CONTINUE = 2
+    END = 3
+    OTHER = 4
+
+class PortalRegStatus(Enum):
+    NO_REG = 0
+    PORTAL = 1
+    SEGA_ID = 2
+
+class ADBStatus(Enum):
+    UNKNOWN = 0
+    GOOD = 1
+    BAD_AMIE_ID = 2
+    ALREADY_REG = 3
+    BAN_SYS_USER = 4
+    BAN_SYS = 5
+    BAN_USER = 6
+    BAN_GEN = 7
+    LOCK_SYS_USER = 8
+    LOCK_SYS = 9
+    LOCK_USER = 10
+
+class CompanyCodes(Enum):
+    NONE = 0
+    SEGA = 1
+    BAMCO = 2
+    KONAMI = 3
+    TAITO = 4
+
+class ReaderFwVer(Enum): # Newer readers use a singly byte value
+    NONE = 0
+    TN32_10 = 1
+    TN32_12 = 2
+    OTHER = 9
+    
+    def __str__(self) -> str:
+        if self == self.TN32_10:
+            return "TN32MSEC003S F/W Ver1.0"
+        elif self == self.TN32_12:
+            return "TN32MSEC003S F/W Ver1.2"
+        elif self == self.NONE:
+            return "Not Specified"
+        elif self == self.OTHER:
+            return "Unknown/Other"
+        else:
+            raise ValueError(f"Bad ReaderFwVer value {self.value}")
+    
+    @classmethod
+    def from_byte(self, byte: bytes) -> Union["ReaderFwVer", int]:
+        try:
+            i = int.from_bytes(byte, 'little')
+            try:
+                return ReaderFwVer(i)
+            except ValueError:
+                return i
+        except TypeError:
+            return 0
+
+class ADBHeaderException(Exception):
+    pass
+
+HEADER_SIZE: Final[int] = 0x20
+CMD_CODE_GOODBYE: Final[int] = 0x66
+
+# everything is LE
+class ADBHeader:
+    def __init__(self, magic: int, protocol_ver: int, cmd: int, length: int, status: int, game_id: Union[str, bytes], store_id: int, keychip_id: Union[str, bytes]) -> None:
+        self.magic = magic # u16
+        self.protocol_ver = protocol_ver # u16
+        self.cmd = cmd # u16
+        self.length = length # u16
+        try:
+            self.status = ADBStatus(status) # u16
+        except ValueError as e:
+            raise ADBHeaderException(f"Status is incorrect! {e}")
+        self.game_id = game_id # 4 char + \x00
+        self.store_id = store_id # u32
+        self.keychip_id = keychip_id# 11 char + \x00
+                
+        if type(self.game_id) == bytes:
+            self.game_id = self.game_id.decode()
+        
+        if type(self.keychip_id) == bytes:
+            self.keychip_id = self.keychip_id.decode()
+        
+        self.game_id = self.game_id.replace("\0", "")
+        self.keychip_id = self.keychip_id.replace("\0", "")
+        if self.cmd != CMD_CODE_GOODBYE: # Games for some reason send no data with goodbye
+            self.validate()
+    
+    @classmethod
+    def from_data(cls, data: bytes) -> "ADBHeader":
+        magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id = struct.unpack_from("<5H6sI12s", data)
+        head = cls(magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id)
+        
+        if head.length > len(data):
+            raise ADBHeaderException(f"Length is incorrect! Expect {head.length}, got {len(data)}")
+        
+        return head
+    
+    def validate(self) -> bool:
+        if self.magic != 0xa13e:
+            raise ADBHeaderException(f"Magic {self.magic} != 0xa13e")
+        
+        if self.protocol_ver < 0x1000:
+            raise ADBHeaderException(f"Protocol version {hex(self.protocol_ver)} is invalid!")
+
+        if re.fullmatch(r"^S[0-9A-Z]{3}[P]?$", self.game_id) is None:
+            raise ADBHeaderException(f"Game ID {self.game_id} is invalid!")
+        
+        if self.store_id == 0:
+            raise ADBHeaderException(f"Store ID cannot be 0!")
+
+        if re.fullmatch(r"^A[0-9]{2}[E|X][0-9]{2}[A-HJ-NP-Z][0-9]{4}$", self.keychip_id) is None:
+            raise ADBHeaderException(f"Keychip ID {self.keychip_id} is invalid!")
+        
+        return True
+
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "magic" / Int16ul,
+            "unknown" / Int16ul,
+            "response_code" / Int16ul,
+            "length" / Int16ul,
+            "status" / Int16ul,
+            "game_id" / PaddedString(6, 'utf_8'),
+            "store_id" / Int32ul,
+            "keychip_id" / PaddedString(12, 'utf_8'),
+        )
+
+        return resp_struct.build(dict(
+            magic=self.magic,
+            unknown=self.protocol_ver,
+            response_code=self.cmd,
+            length=self.length,
+            status=self.status.value,
+            game_id = self.game_id,
+            store_id = self.store_id,
+            keychip_id = self.keychip_id,
+        ))
+
+class ADBBaseRequest:
+    def __init__(self, data: bytes) -> None:
+        self.head = ADBHeader.from_data(data)
+
+class ADBBaseResponse:
+    def __init__(self, code: int = 0, length: int = 0x20, status: int = 1, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", protocol_ver: int = 0x3087) -> None:
+        self.head = ADBHeader(0xa13e, protocol_ver, code, length, status, game_id, store_id, keychip_id)
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, cmd: int, length: int = 0x20, status: int = 1) -> "ADBBaseResponse":
+        return cls(cmd, length, status, req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
+
+    def append_padding(self, data: bytes):
+        """Appends 0s to the end of the data until it's at the correct size"""
+        padding_size = self.head.length - len(data)
+        data += bytes(padding_size)
+        return data
+
+    def make(self) -> bytes:
+        return self.head.make()

core/adb_handlers/campaign.py

@@ -0,0 +1,132 @@
+from construct import Struct, Int16ul, Padding, Bytes, Int32ul, Int32sl
+
+from .base import *
+
+class Campaign:
+    def __init__(self) -> None:
+        self.id = 0
+        self.name = ""
+        self.announce_date = 0
+        self.start_date = 0
+        self.end_date = 0
+        self.distrib_start_date = 0
+        self.distrib_end_date = 0
+    
+    def make(self) -> bytes:
+        name_padding = bytes(128 - len(self.name))
+        return Struct(
+            "id" / Int32ul,
+            "name" / Bytes(128),
+            "announce_date" / Int32ul,
+            "start_date" / Int32ul,
+            "end_date" / Int32ul,
+            "distrib_start_date" / Int32ul,
+            "distrib_end_date" / Int32ul,
+            Padding(8),
+        ).build(dict(
+            id = self.id,
+            name = self.name.encode() + name_padding,
+            announce_date = self.announce_date,
+            start_date = self.start_date,
+            end_date = self.end_date,
+            distrib_start_date = self.distrib_start_date,
+            distrib_end_date = self.distrib_end_date,
+        ))
+
+class CampaignClear:
+    def __init__(self) -> None:
+        self.id = 0
+        self.entry_flag = 0
+        self.clear_flag = 0
+    
+    def make(self) -> bytes:
+        return Struct(
+            "id" / Int32ul,
+            "entry_flag" / Int32ul,
+            "clear_flag" / Int32ul,
+            Padding(4),
+        ).build(dict(
+            id = self.id,
+            entry_flag = self.entry_flag,
+            clear_flag = self.clear_flag,
+        ))
+
+class ADBCampaignResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0C, length: int = 0x200, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.campaigns = [Campaign(), Campaign(), Campaign()]
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self) -> bytes:
+        body = b""
+        
+        for c in self.campaigns:
+            body += c.make()
+        
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body
+
+class ADBOldCampaignRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.campaign_id = struct.unpack_from("<I", data, 0x20)
+
+class ADBOldCampaignResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0C, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.info0 = 0
+        self.info1 = 0
+        self.info2 = 0
+        self.info3 = 0
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "info0" / Int32sl,
+            "info1" / Int32sl,
+            "info2" / Int32sl,
+            "info3" / Int32sl,
+        ).build(
+            info0 = self.info0,
+            info1 = self.info1,
+            info2 = self.info2,
+            info3 = self.info3,
+        )
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+        return self.head.make() + resp_struct
+
+class ADBCampaignClearRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id = struct.unpack_from("<i", data, 0x20)
+
+class ADBCampaignClearResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0E, length: int = 0x50, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.campaign_clear_status = [CampaignClear(), CampaignClear(), CampaignClear()]
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:
+        body = b""
+        
+        for c in self.campaign_clear_status:
+            body += c.make()
+        
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/adb_handlers/felica.py

@@ -0,0 +1,85 @@
+from construct import Struct, Int32sl, Padding, Int8ub, Int16sl
+from typing import Union
+from .base import *
+
+class ADBFelicaLookupRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        idm, pmm = struct.unpack_from(">QQ", data, 0x20)
+        self.idm = hex(idm)[2:].upper()
+        self.pmm = hex(pmm)[2:].upper()
+
+class ADBFelicaLookupResponse(ADBBaseResponse):
+    def __init__(self, access_code: str = None, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x03, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.access_code = access_code if access_code is not None else "00000000000000000000"
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader, access_code: str = None) -> "ADBFelicaLookupResponse":
+        c = cls(access_code, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:        
+        resp_struct = Struct(
+            "felica_idx" / Int32ul,
+            "access_code" / Int8ub[10],
+            Padding(2)
+        ).build(dict(
+            felica_idx = 0,
+            access_code = bytes.fromhex(self.access_code)
+        ))
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+
+        return self.head.make() + resp_struct
+
+class ADBFelicaLookup2Request(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.random = struct.unpack_from("<16s", data, 0x20)[0]
+        idm, pmm = struct.unpack_from(">QQ", data, 0x30)
+        self.card_key_ver, self.write_ct, self.maca, company, fw_ver, self.dfc = struct.unpack_from("<16s16sQccH", data, 0x40)
+        self.idm = hex(idm)[2:].upper()
+        self.pmm = hex(pmm)[2:].upper()
+        self.company = CompanyCodes(int.from_bytes(company, 'little'))
+        self.fw_ver = ReaderFwVer.from_byte(fw_ver)
+
+class ADBFelicaLookup2Response(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None] = None, access_code: Union[str, None] = None, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x12, length: int = 0x130, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.access_code = access_code if access_code is not None else "00000000000000000000"
+        self.company = CompanyCodes.SEGA
+        self.portal_status = PortalRegStatus.NO_REG
+        self.auth_key = [0] * 256
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None] = None, access_code: Union[str, None] = None) -> "ADBFelicaLookup2Response":
+        c = cls(user_id, access_code, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self) -> bytes:        
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "relation1" / Int32sl,
+            "relation2" / Int32sl,
+            "access_code" / Int8ub[10],
+            "portal_status" / Int8ub,
+            "company_code" / Int8ub,
+            Padding(8),
+            "auth_key" / Int8ub[256],
+        ).build(dict(
+            user_id = self.user_id,
+            relation1 = -1, # Unsupported
+            relation2 = -1, # Unsupported
+            access_code = bytes.fromhex(self.access_code),
+            portal_status = self.portal_status.value,
+            company_code = self.company.value,
+            auth_key = self.auth_key
+        ))
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+
+        return self.head.make() + resp_struct

core/adb_handlers/log.py

@@ -0,0 +1,56 @@
+from construct import Struct, Padding, Int8sl
+from typing import Final, List
+
+from .base import *
+NUM_LOGS: Final[int] = 20
+NUM_LEN_LOG_EX: Final[int] = 48
+
+class AmLogEx:
+    def __init__(self, data: bytes) -> None:
+        self.aime_id, status, self.user_id, self.credit_ct, self.bet_ct, self.won_ct, self.local_time, \
+            self.tseq, self.place_id = struct.unpack("<IIQiii4xQiI", data)
+        self.status = LogStatus(status)
+
+class ADBStatusLogRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id, status = struct.unpack_from("<II", data, 0x20)
+        self.status = LogStatus(status)
+
+class ADBLogRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id, status, self.user_id, self.credit_ct, self.bet_ct, self.won_ct = struct.unpack_from("<IIQiii", data, 0x20)
+        self.status = LogStatus(status)
+
+class ADBLogExRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.logs: List[AmLogEx] = []
+
+        for x in range(NUM_LOGS):
+            self.logs.append(AmLogEx(data[0x20 + (NUM_LEN_LOG_EX * x): 0x50 + (NUM_LEN_LOG_EX * x)]))
+        
+        self.num_logs = struct.unpack_from("<I", data, 0x03E0)[0]
+
+class ADBLogExResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", protocol_ver: int = 12423, code: int = 20, length: int = 64, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id, protocol_ver)
+
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBLogExResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
+        return c
+    
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "log_result" / Int8sl[NUM_LOGS],
+            Padding(12)
+        )
+
+        body = resp_struct.build(dict(
+            log_result = [1] * NUM_LOGS
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/adb_handlers/lookup.py

@@ -0,0 +1,82 @@
+from construct import Struct, Int32sl, Padding, Int8sl
+from typing import Union
+
+from .base import *
+
+class ADBLookupException(Exception):
+    pass
+
+class ADBLookupRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.access_code = data[0x20:0x2A].hex()
+        company_code, fw_version, self.serial_number = struct.unpack_from("<bbI", data, 0x2A)
+        
+        try:
+            self.company_code = CompanyCodes(company_code)
+        except ValueError as e:
+            raise ADBLookupException(f"Invalid company code - {e}")
+        
+        self.fw_version = ReaderFwVer.from_byte(fw_version)
+
+class ADBLookupResponse(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None], game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x06, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.portal_reg = PortalRegStatus.NO_REG
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None]) -> "ADBLookupResponse":
+        c = cls(user_id, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self):
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "portal_reg" / Int8sl,
+            Padding(11)
+        )
+
+        body = resp_struct.build(dict(
+            user_id = self.user_id,
+            portal_reg = self.portal_reg.value
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body
+
+class ADBLookupExResponse(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None], game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", 
+                code: int = 0x10, length: int = 0x130, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.portal_reg = PortalRegStatus.NO_REG
+        self.auth_key = [0] * 256
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None]) -> "ADBLookupExResponse":
+        c = cls(user_id, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self):
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "portal_reg" / Int8sl,
+            Padding(3),
+            "auth_key" / Int8sl[256],
+            "relation1" / Int32sl,
+            "relation2" / Int32sl,
+        )
+
+        body = resp_struct.build(dict(
+            user_id = self.user_id,
+            portal_reg = self.portal_reg.value,
+            auth_key = self.auth_key,
+            relation1 = -1,
+            relation2 = -1
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/aimedb.py

@@ -1,297 +1,369 @@
-from twisted.internet.protocol import Factory, Protocol
 import logging, coloredlogs
 from Crypto.Cipher import AES
-import struct
-from typing import Dict, Any
+from typing import Dict, Tuple, Callable, Union, Optional
+import asyncio
 from logging.handlers import TimedRotatingFileHandler
 
 from core.config import CoreConfig
+from core.utils import create_sega_auth_key
 from core.data import Data
+from .adb_handlers import *
 
-
-class AimedbProtocol(Protocol):
-    AIMEDB_RESPONSE_CODES = {
-        "felica_lookup": 0x03,
-        "lookup": 0x06,
-        "log": 0x0A,
-        "campaign": 0x0C,
-        "touch": 0x0E,
-        "lookup2": 0x10,
-        "felica_lookup2": 0x12,
-        "log2": 0x14,
-        "hello": 0x65,
-    }
-
-    request_list: Dict[int, Any] = {}
-
-    def __init__(self, core_cfg: CoreConfig) -> None:
-        self.logger = logging.getLogger("aimedb")
-        self.config = core_cfg
+class AimedbServlette():
+    request_list: Dict[int, Tuple[Callable[[bytes, int], Union[ADBBaseResponse, bytes]], int, str]] = {}
+    def __init__(self, core_cfg: CoreConfig) -> None:        
+        self.config = core_cfg        
         self.data = Data(core_cfg)
-        if core_cfg.aimedb.key == "":
+
+        self.logger = logging.getLogger("aimedb")
+        if not hasattr(self.logger, "initted"):
+            log_fmt_str = "[%(asctime)s] Aimedb | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.config.server.log_dir, "aimedb"),
+                when="d",
+                backupCount=10,
+            )
+            fileHandler.setFormatter(log_fmt)
+
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
+
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
+
+            self.logger.setLevel(self.config.aimedb.loglevel)
+            coloredlogs.install(
+                level=core_cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.initted = True
+
+        if not core_cfg.aimedb.key:
             self.logger.error("!!!KEY NOT SET!!!")
             exit(1)
 
-        self.request_list[0x01] = self.handle_felica_lookup
-        self.request_list[0x04] = self.handle_lookup
-        self.request_list[0x05] = self.handle_register
-        self.request_list[0x09] = self.handle_log
-        self.request_list[0x0B] = self.handle_campaign
-        self.request_list[0x0D] = self.handle_touch
-        self.request_list[0x0F] = self.handle_lookup2
-        self.request_list[0x11] = self.handle_felica_lookup2
-        self.request_list[0x13] = self.handle_log2
-        self.request_list[0x64] = self.handle_hello
+        self.register_handler(0x01, 0x03, self.handle_felica_lookup, 'felica_lookup')
+        self.register_handler(0x02, 0x03, self.handle_felica_register, 'felica_register')
 
-    def append_padding(self, data: bytes):
-        """Appends 0s to the end of the data until it's at the correct size"""
-        length = struct.unpack_from("<H", data, 6)
-        padding_size = length[0] - len(data)
-        data += bytes(padding_size)
-        return data
+        self.register_handler(0x04, 0x06, self.handle_lookup, 'lookup')
+        self.register_handler(0x05, 0x06, self.handle_register, 'register')
 
-    def connectionMade(self) -> None:
-        self.logger.debug(f"{self.transport.getPeer().host} Connected")
+        self.register_handler(0x07, 0x08, self.handle_status_log, 'status_log')
+        self.register_handler(0x09, 0x0A, self.handle_log, 'aime_log')       
 
-    def connectionLost(self, reason) -> None:
-        self.logger.debug(
-            f"{self.transport.getPeer().host} Disconnected - {reason.value}"
-        )
+        self.register_handler(0x0B, 0x0C, self.handle_campaign, 'campaign')
+        self.register_handler(0x0D, 0x0E, self.handle_campaign_clear, 'campaign_clear')
 
-    def dataReceived(self, data: bytes) -> None:
+        self.register_handler(0x0F, 0x10, self.handle_lookup_ex, 'lookup_ex')
+        self.register_handler(0x11, 0x12, self.handle_felica_lookup_ex, 'felica_lookup_ex')
+
+        self.register_handler(0x13, 0x14, self.handle_log_ex, 'aime_log_ex')
+        self.register_handler(0x64, 0x65, self.handle_hello, 'hello')
+
+    def register_handler(self, cmd: int, resp:int, handler: Callable[[bytes, int], Union[ADBBaseResponse, bytes]], name: str) -> None:
+        self.request_list[cmd] = (handler, resp, name)
+    
+    def start(self) -> None:
+        self.logger.info(f"Start on port {self.config.aimedb.port}")
+        addr = self.config.aimedb.listen_address if self.config.aimedb.listen_address else self.config.server.listen_address
+        asyncio.create_task(asyncio.start_server(self.dataReceived, addr, self.config.aimedb.port))
+    
+    async def dataReceived(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
+        self.logger.debug(f"Connection made from {writer.get_extra_info('peername')[0]}")
+        while True:
+            try:
+                data: bytes = await reader.read(4096)
+                if len(data) == 0:
+                    self.logger.debug("Connection closed")
+                    return
+                await self.process_data(data, reader, writer)
+                await writer.drain()
+            except ConnectionResetError as e:
+                self.logger.debug("Connection reset, disconnecting")
+                return
+
+    async def process_data(self, data: bytes, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> Optional[bytes]:
+        addr = writer.get_extra_info('peername')[0]
         cipher = AES.new(self.config.aimedb.key.encode(), AES.MODE_ECB)
 
         try:
             decrypted = cipher.decrypt(data)
-        except:
-            self.logger.error(f"Failed to decrypt {data.hex()}")
-            return None
-
-        self.logger.debug(f"{self.transport.getPeer().host} wrote {decrypted.hex()}")
-
-        if not decrypted[1] == 0xA1 and not decrypted[0] == 0x3E:
-            self.logger.error(f"Bad magic")
-            return None
-
-        req_code = decrypted[4]
-
-        if req_code == 0x66:
-            self.logger.info(f"goodbye from {self.transport.getPeer().host}")
-            self.transport.loseConnection()
+        
+        except Exception as e:
+            self.logger.error(f"Failed to decrypt {data.hex()} because {e}")
             return
 
+        self.logger.debug(f"{addr} wrote {decrypted.hex()}")
+
         try:
-            resp = self.request_list[req_code](decrypted)
-            encrypted = cipher.encrypt(resp)
-            self.logger.debug(f"Response {resp.hex()}")
-            self.transport.write(encrypted)
+            head = ADBHeader.from_data(decrypted)
+        
+        except ADBHeaderException as e:
+            self.logger.error(f"Error parsing ADB header: {e}")
+            try:    
+                encrypted = cipher.encrypt(ADBBaseResponse().make())
+                writer.write(encrypted)
+                await writer.drain()
+                return
 
-        except KeyError:
-            self.logger.error(f"Unknown command code {hex(req_code)}")
-            return None
+            except Exception as e:
+                self.logger.error(f"Failed to encrypt default response because {e}")
+            
+            return
 
-        except ValueError as e:
-            self.logger.error(f"Failed to encrypt {resp.hex()} because {e}")
-            return None
+        if head.keychip_id == "ABCD1234567" or head.store_id == 0xfff0:
+            self.logger.warning(f"Request from uninitialized AMLib: {vars(head)}")
 
-    def handle_campaign(self, data: bytes) -> bytes:
-        self.logger.info(f"campaign from {self.transport.getPeer().host}")
-        ret = struct.pack(
-            "<5H",
-            0xA13E,
-            0x3087,
-            self.AIMEDB_RESPONSE_CODES["campaign"],
-            0x0200,
-            0x0001,
-        )
-        return self.append_padding(ret)
+        if head.cmd == 0x66:
+            self.logger.info("Goodbye")
+            writer.close()
+            return
 
-    def handle_hello(self, data: bytes) -> bytes:
-        self.logger.info(f"hello from {self.transport.getPeer().host}")
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["hello"], 0x0020, 0x0001
-        )
-        return self.append_padding(ret)
+        handler, resp_code, name = self.request_list.get(head.cmd, (self.handle_default, None, 'default'))
 
-    def handle_lookup(self, data: bytes) -> bytes:
-        luid = data[0x20:0x2A].hex()
-        user_id = self.data.card.get_user_id_from_card(access_code=luid)
+        if resp_code is None:
+            self.logger.warning(f"No handler for cmd {hex(head.cmd)}")
+        
+        elif resp_code > 0:
+            self.logger.info(f"{name} from {head.keychip_id} ({head.game_id}) @ {addr}")
+        
+        resp = await handler(decrypted, resp_code)
 
-        if user_id is None:
-            user_id = -1
+        if type(resp) == ADBBaseResponse or issubclass(type(resp), ADBBaseResponse):
+            resp_bytes = resp.make()
 
-        self.logger.info(
-            f"lookup from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}"
-        )
-
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["lookup"], 0x0130, 0x0001
-        )
-        ret += bytes(0x20 - len(ret))
-
-        if user_id is None:
-            ret += struct.pack("<iH", -1, 0)
+        elif type(resp) == bytes:
+            resp_bytes = resp
+        
+        elif resp is None: # Nothing to send, probably a goodbye
+            self.logger.warn(f"None return by handler for {name}")
+            return
+        
         else:
-            ret += struct.pack("<l", user_id)
-        return self.append_padding(ret)
+            self.logger.error(f"Unsupported type returned by ADB handler for {name}: {type(resp)}")
+            raise TypeError(f"Unsupported type returned by ADB handler for {name}: {type(resp)}")
 
-    def handle_lookup2(self, data: bytes) -> bytes:
-        self.logger.info(f"lookup2")
+        try:
+            encrypted = cipher.encrypt(resp_bytes)
+            self.logger.debug(f"Response {resp_bytes.hex()}")
+            writer.write(encrypted)
 
-        ret = bytearray(self.handle_lookup(data))
-        ret[4] = self.AIMEDB_RESPONSE_CODES["lookup2"]
+        except Exception as e:
+            self.logger.error(f"Failed to encrypt {resp_bytes.hex()} because {e}")
+    
+    async def handle_default(self, data: bytes, resp_code: int, length: int = 0x20) -> ADBBaseResponse:
+        req = ADBHeader.from_data(data)
+        return ADBBaseResponse(resp_code, length, 1, req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
 
-        return bytes(ret)
+    async def handle_hello(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        return await self.handle_default(data, resp_code)
 
-    def handle_felica_lookup(self, data: bytes) -> bytes:
-        idm = data[0x20:0x28].hex()
-        pmm = data[0x28:0x30].hex()
-        access_code = self.data.card.to_access_code(idm)
+    async def handle_campaign(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        h = ADBHeader.from_data(data)
+        if h.protocol_ver >= 0x3030:
+            req = h
+            resp = ADBCampaignResponse.from_req(req)
+
+        else:
+            req = ADBOldCampaignRequest(data)
+            
+            self.logger.info(f"Legacy campaign request for campaign {req.campaign_id} (protocol version {hex(h.protocol_ver)})")
+            resp = ADBOldCampaignResponse.from_req(req.head)
+        
+        # We don't currently support campaigns
+        return resp
+
+    async def handle_lookup(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        req = ADBLookupRequest(data)
+        user_id = await self.data.card.get_user_id_from_card(req.access_code)
+        is_banned = await self.data.card.get_card_banned(req.access_code)
+        is_locked = await self.data.card.get_card_locked(req.access_code)
+        
+        ret = ADBLookupResponse.from_req(req.head, user_id)
+        if is_banned and is_locked:
+            ret.head.status = ADBStatus.BAN_SYS_USER
+        elif is_banned:
+            ret.head.status = ADBStatus.BAN_SYS
+        elif is_locked:
+            ret.head.status = ADBStatus.LOCK_USER
+        
         self.logger.info(
-            f"felica_lookup from {self.transport.getPeer().host}: idm {idm} pmm {pmm} -> access_code {access_code}"
+            f"access_code {req.access_code} -> user_id {ret.user_id}"
         )
+        
+        if user_id and user_id > 0:
+            await self.data.card.update_card_last_login(req.access_code)
+        return ret
 
-        ret = struct.pack(
-            "<5H",
-            0xA13E,
-            0x3087,
-            self.AIMEDB_RESPONSE_CODES["felica_lookup"],
-            0x0030,
-            0x0001,
-        )
-        ret += bytes(26)
-        ret += bytes.fromhex(access_code)
+    async def handle_lookup_ex(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        req = ADBLookupRequest(data)
+        user_id = await self.data.card.get_user_id_from_card(req.access_code)
 
-        return self.append_padding(ret)
+        is_banned = await self.data.card.get_card_banned(req.access_code)
+        is_locked = await self.data.card.get_card_locked(req.access_code)
 
-    def handle_felica_lookup2(self, data: bytes) -> bytes:
-        idm = data[0x30:0x38].hex()
-        pmm = data[0x38:0x40].hex()
-        access_code = self.data.card.to_access_code(idm)
-        user_id = self.data.card.get_user_id_from_card(access_code=access_code)
-
-        if user_id is None:
-            user_id = -1
+        ret = ADBLookupExResponse.from_req(req.head, user_id)
+        if is_banned and is_locked:
+            ret.head.status = ADBStatus.BAN_SYS_USER
+        elif is_banned:
+            ret.head.status = ADBStatus.BAN_SYS
+        elif is_locked:
+            ret.head.status = ADBStatus.LOCK_USER
 
         self.logger.info(
-            f"felica_lookup2 from {self.transport.getPeer().host}: idm {idm} ipm {pmm} -> access_code {access_code} user_id {user_id}"
+            f"access_code {req.access_code} -> user_id {ret.user_id}"
         )
 
-        ret = struct.pack(
-            "<5H",
-            0xA13E,
-            0x3087,
-            self.AIMEDB_RESPONSE_CODES["felica_lookup2"],
-            0x0140,
-            0x0001,
+        if user_id and user_id > 0 and self.config.aimedb.id_secret:
+            auth_key = create_sega_auth_key(user_id, req.head.game_id, req.head.store_id, req.head.keychip_id, self.config.aimedb.id_secret, self.config.aimedb.id_lifetime_seconds)
+            if auth_key is not None:
+                auth_key_extra_len = 256 - len(auth_key)
+                auth_key_full = auth_key.encode() + (b"\0" * auth_key_extra_len)
+                self.logger.debug(f"Generated auth token {auth_key}")
+                ret.auth_key = auth_key_full
+
+        if user_id and user_id > 0:
+            await self.data.card.update_card_last_login(req.access_code)
+        return ret
+
+    async def handle_felica_lookup(self, data: bytes, resp_code: int) -> bytes:
+        """
+        On official, I think a card has to be registered for this to actually work, but 
+        I'm making the executive decision to not implement that and just kick back our
+        faux generated access code. The real felica IDm -> access code conversion is done
+        on the ADB server, which we do not and will not ever have access to. Because we can
+        assure that all IDms will be unique, this basic 0-padded hex -> int conversion will
+        be fine.
+        """
+        req = ADBFelicaLookupRequest(data)
+        ac = self.data.card.to_access_code(req.idm)
+        self.logger.info(
+            f"idm {req.idm} ipm {req.pmm} -> access_code {ac}"
         )
-        ret += bytes(22)
-        ret += struct.pack("<lq", user_id, -1)  # first -1 is ext_id, 3rd is access code
-        ret += bytes.fromhex(access_code)
-        ret += struct.pack("<l", 1)
+        return ADBFelicaLookupResponse.from_req(req.head, ac)
 
-        return self.append_padding(ret)
-
-    def handle_touch(self, data: bytes) -> bytes:
-        self.logger.info(f"touch from {self.transport.getPeer().host}")
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["touch"], 0x0050, 0x0001
-        )
-        ret += bytes(5)
-        ret += struct.pack("<3H", 0x6F, 0, 1)
-
-        return self.append_padding(ret)
-
-    def handle_register(self, data: bytes) -> bytes:
-        luid = data[0x20:0x2A].hex()
+    async def handle_felica_register(self, data: bytes, resp_code: int) -> bytes:
+        """
+        I've never seen this used.
+        """
+        req = ADBFelicaLookupRequest(data)
+        ac = self.data.card.to_access_code(req.idm)
+        
         if self.config.server.allow_user_registration:
-            user_id = self.data.user.create_user()
+            user_id = await self.data.user.create_user()
 
             if user_id is None:
-                user_id = -1
                 self.logger.error("Failed to register user!")
+                user_id = -1
 
             else:
-                card_id = self.data.card.create_card(user_id, luid)
+                card_id = await self.data.card.create_card(user_id, ac)
 
                 if card_id is None:
-                    user_id = -1
                     self.logger.error("Failed to register card!")
+                    user_id = -1
 
             self.logger.info(
-                f"register from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}"
+                f"Register access code {ac} (IDm: {req.idm} PMm: {req.pmm}) -> user_id {user_id}"
             )
 
         else:
             self.logger.info(
-                f"register from {self.transport.getPeer().host} blocked!: luid {luid}"
+                f"Registration blocked!: access code {ac} (IDm: {req.idm} PMm: {req.pmm})"
             )
+
+        if user_id > 0:
+            await self.data.card.update_card_last_login(ac)
+        return ADBFelicaLookupResponse.from_req(req.head, ac)
+
+    async def handle_felica_lookup_ex(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBFelicaLookup2Request(data)
+        access_code = self.data.card.to_access_code(req.idm)
+        user_id = await self.data.card.get_user_id_from_card(access_code=access_code)
+
+        if user_id is None:
             user_id = -1
 
-        ret = struct.pack(
-            "<5H",
-            0xA13E,
-            0x3087,
-            self.AIMEDB_RESPONSE_CODES["lookup"],
-            0x0030,
-            0x0001 if user_id > -1 else 0,
-        )
-        ret += bytes(0x20 - len(ret))
-        ret += struct.pack("<l", user_id)
-
-        return self.append_padding(ret)
-
-    def handle_log(self, data: bytes) -> bytes:
-        # TODO: Save aimedb logs
-        self.logger.info(f"log from {self.transport.getPeer().host}")
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["log"], 0x0020, 0x0001
-        )
-        return self.append_padding(ret)
-
-    def handle_log2(self, data: bytes) -> bytes:
-        self.logger.info(f"log2 from {self.transport.getPeer().host}")
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["log2"], 0x0040, 0x0001
-        )
-        ret += bytes(22)
-        ret += struct.pack("H", 1)
-
-        return self.append_padding(ret)
-
-
-class AimedbFactory(Factory):
-    protocol = AimedbProtocol
-
-    def __init__(self, cfg: CoreConfig) -> None:
-        self.config = cfg
-        log_fmt_str = "[%(asctime)s] Aimedb | %(levelname)s | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
-        self.logger = logging.getLogger("aimedb")
-
-        fileHandler = TimedRotatingFileHandler(
-            "{0}/{1}.log".format(self.config.server.log_dir, "aimedb"),
-            when="d",
-            backupCount=10,
-        )
-        fileHandler.setFormatter(log_fmt)
-
-        consoleHandler = logging.StreamHandler()
-        consoleHandler.setFormatter(log_fmt)
-
-        self.logger.addHandler(fileHandler)
-        self.logger.addHandler(consoleHandler)
-
-        self.logger.setLevel(self.config.aimedb.loglevel)
-        coloredlogs.install(
-            level=cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str
+        self.logger.info(
+            f"idm {req.idm} ipm {req.pmm} -> access_code {access_code} user_id {user_id}"
         )
 
-        if self.config.aimedb.key == "":
-            self.logger.error("Please set 'key' field in your config file.")
-            exit(1)
+        resp = ADBFelicaLookup2Response.from_req(req.head, user_id, access_code)
 
-        self.logger.info(f"Ready on port {self.config.aimedb.port}")
+        if user_id and user_id > 0 and self.config.aimedb.id_secret:
+            auth_key = create_sega_auth_key(user_id, req.head.game_id, req.head.store_id, req.head.keychip_id, self.config.aimedb.id_secret, self.config.aimedb.id_lifetime_seconds)
+            if auth_key is not None:
+                auth_key_extra_len = 256 - len(auth_key)
+                auth_key_full = auth_key.encode() + (b"\0" * auth_key_extra_len)
+                self.logger.debug(f"Generated auth token {auth_key}")
+                resp.auth_key = auth_key_full
+        
+        if user_id and user_id > 0:
+            await self.data.card.update_card_last_login(access_code)
+        return resp
+
+    async def handle_campaign_clear(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        req = ADBCampaignClearRequest(data)
+
+        resp = ADBCampaignClearResponse.from_req(req.head)
+
+        # We don't support campaign stuff
+        return resp
+
+    async def handle_register(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBLookupRequest(data)
+        user_id = -1
+
+        if self.config.server.allow_user_registration:
+            user_id = await self.data.user.create_user()
+
+            if user_id is None:
+                self.logger.error("Failed to register user!")
+                user_id = -1
+
+            else:
+                card_id = await self.data.card.create_card(user_id, req.access_code)
+
+                if card_id is None:
+                    self.logger.error("Failed to register card!")
+                    user_id = -1
+
+            self.logger.info(
+                f"Register access code {req.access_code} -> user_id {user_id}"
+            )
+
+        else:
+            self.logger.info(
+                f"Registration blocked!: access code {req.access_code}"
+            )
+
+        resp = ADBLookupResponse.from_req(req.head, user_id)
+        if resp.user_id <= 0:
+            resp.head.status = ADBStatus.BAN_SYS # Closest we can get to a "You cannot register"
+
+        else:
+            await self.data.card.update_card_last_login(req.access_code)
+
+        return resp
+
+    # TODO: Save these in some capacity, as deemed relevant
+    async def handle_status_log(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBStatusLogRequest(data)
+        self.logger.info(f"User {req.aime_id} logged {req.status.name} event")
+        return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
+
+    async def handle_log(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBLogRequest(data)
+        self.logger.info(f"User {req.aime_id} logged {req.status.name} event, credit_ct: {req.credit_ct} bet_ct: {req.bet_ct} won_ct: {req.won_ct}")
+        return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
+
+    async def handle_log_ex(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBLogExRequest(data)
+        strs = []
+        self.logger.info(f"Recieved {req.num_logs} or {len(req.logs)} logs")
+        
+        for x in range(req.num_logs):
+            self.logger.debug(f"User {req.logs[x].aime_id} logged {req.logs[x].status.name} event, credit_ct: {req.logs[x].credit_ct} bet_ct: {req.logs[x].bet_ct} won_ct: {req.logs[x].won_ct}")
+        return ADBLogExResponse.from_req(req.head)
 
-    def buildProtocol(self, addr):
-        return AimedbProtocol(self.config)

core/app.py

@@ -0,0 +1,92 @@
+import yaml
+import logging
+import coloredlogs
+from logging.handlers import TimedRotatingFileHandler
+from starlette.routing import Route
+from starlette.requests import Request
+from starlette.applications import Starlette
+from starlette.responses import PlainTextResponse
+from os import environ, path, mkdir, W_OK, access
+from typing import List
+
+from core import CoreConfig, TitleServlet, MuchaServlet, AllnetServlet, BillingServlet, AimedbServlette
+from core.frontend import FrontendServlet
+
+async def dummy_rt(request: Request):
+    return PlainTextResponse("Service OK")
+
+cfg_dir = environ.get("ARTEMIS_CFG_DIR", "config")
+cfg: CoreConfig = CoreConfig()
+if path.exists(f"{cfg_dir}/core.yaml"):
+    cfg.update(yaml.safe_load(open(f"{cfg_dir}/core.yaml")))
+
+if not path.exists(cfg.server.log_dir):
+    mkdir(cfg.server.log_dir)
+
+if not access(cfg.server.log_dir, W_OK):
+    print(
+        f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
+    )
+    exit(1)
+
+logger = logging.getLogger("core")
+log_fmt_str = "[%(asctime)s] Core | %(levelname)s | %(message)s"
+log_fmt = logging.Formatter(log_fmt_str)
+
+fileHandler = TimedRotatingFileHandler(
+    "{0}/{1}.log".format(cfg.server.log_dir, "core"), when="d", backupCount=10
+)
+fileHandler.setFormatter(log_fmt)
+
+consoleHandler = logging.StreamHandler()
+consoleHandler.setFormatter(log_fmt)
+
+logger.addHandler(fileHandler)
+logger.addHandler(consoleHandler)
+
+log_lv = logging.DEBUG if cfg.server.is_develop else logging.INFO
+logger.setLevel(log_lv)
+coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
+
+logger.info(f"Artemis starting in {'develop' if cfg.server.is_develop else 'production'} mode")
+
+title = TitleServlet(cfg, cfg_dir) # This has to be loaded first to load plugins
+mucha = MuchaServlet(cfg, cfg_dir)
+
+route_lst: List[Route] = [
+    # Mucha
+    Route("/mucha_front/boardauth.do", mucha.handle_boardauth, methods=["POST"]),
+    Route("/mucha_front/updatacheck.do", mucha.handle_updatecheck, methods=["POST"]),
+    Route("/mucha_front/downloadstate.do", mucha.handle_dlstate, methods=["POST"]),
+    # General
+    Route("/", dummy_rt),
+    Route("/robots.txt", FrontendServlet.robots)
+]
+
+if not cfg.billing.standalone:
+    billing = BillingServlet(cfg, cfg_dir)
+    route_lst += [
+        Route("/request", billing.handle_billing_request, methods=["POST"]),
+        Route("/request/", billing.handle_billing_request, methods=["POST"]),
+    ]
+
+if not cfg.allnet.standalone:
+    allnet = AllnetServlet(cfg, cfg_dir)
+    route_lst += [
+        Route("/sys/servlet/PowerOn", allnet.handle_poweron, methods=["GET", "POST"]),
+        Route("/sys/servlet/DownloadOrder", allnet.handle_dlorder, methods=["GET", "POST"]),
+        Route("/sys/servlet/LoaderStateRecorder", allnet.handle_loaderstaterecorder, methods=["GET", "POST"]),
+        Route("/sys/servlet/Alive", allnet.handle_alive, methods=["GET", "POST"]),
+        Route("/naomitest.html", allnet.handle_naomitest),
+    ]
+    
+    if cfg.allnet.allow_online_updates:
+        route_lst += [
+            Route("/report-api/Report", allnet.handle_dlorder_report, methods=["POST"]),
+            Route("/dl/ini/{file:str}", allnet.handle_dlorder_ini),
+        ]
+
+for code, game in title.title_registry.items():
+    route_lst += game.get_routes()
+
+app = Starlette(cfg.server.is_develop, route_lst)

core/config.py

@@ -1,16 +1,48 @@
 import logging, os
 from typing import Any
 
-
 class ServerConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
     def listen_address(self) -> str:
+        """
+        Address Artemis will bind to and listen on
+        """
         return CoreConfig.get_config_field(
             self.__config, "core", "server", "listen_address", default="127.0.0.1"
         )
+    
+    @property
+    def hostname(self) -> str:
+        """
+        Hostname sent to games
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "hostname", default="localhost"
+        )
+    
+    @property
+    def port(self) -> int:
+        """
+        Port the game will listen on
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "port", default=80
+        )
+
+    @property
+    def ssl_key(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "ssl_key", default="cert/title.key"
+        )
+
+    @property
+    def ssl_cert(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "ssl_cert", default="cert/title.pem"
+        )
 
     @property
     def allow_user_registration(self) -> bool:
@@ -36,12 +68,49 @@ class ServerConfig:
             self.__config, "core", "server", "is_develop", default=True
         )
 
+    @property
+    def is_using_proxy(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "is_using_proxy", default=False
+        )
+
+    @property
+    def proxy_port(self) -> int:
+        """
+        What port the proxy is listening on. This will be sent instead of 'port' if 
+        is_using_proxy is True and this value is non-zero
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "proxy_port", default=0
+        )
+
+    @property
+    def proxy_port_ssl(self) -> int:
+        """
+        What port the proxy is listening for secure connections on. This will be sent 
+        instead of 'port' if is_using_proxy is True and this value is non-zero
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "proxy_port_ssl", default=0
+        )
+
     @property
     def log_dir(self) -> str:
         return CoreConfig.get_config_field(
             self.__config, "core", "server", "log_dir", default="logs"
         )
 
+    @property
+    def check_arcade_ip(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "check_arcade_ip", default=False
+        )
+
+    @property
+    def strict_ip_checking(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "strict_ip_checking", default=False
+        )
 
 class TitleConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -56,18 +125,17 @@ class TitleConfig:
         )
 
     @property
-    def hostname(self) -> str:
+    def reboot_start_time(self) -> str:
         return CoreConfig.get_config_field(
-            self.__config, "core", "title", "hostname", default="localhost"
+            self.__config, "core", "title", "reboot_start_time", default=""
         )
 
     @property
-    def port(self) -> int:
+    def reboot_end_time(self) -> str:
         return CoreConfig.get_config_field(
-            self.__config, "core", "title", "port", default=8080
+            self.__config, "core", "title", "reboot_end_time", default=""
         )
 
-
 class DatabaseConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
@@ -105,7 +173,7 @@ class DatabaseConfig:
     @property
     def protocol(self) -> str:
         return CoreConfig.get_config_field(
-            self.__config, "core", "database", "type", default="mysql"
+            self.__config, "core", "database", "protocol", default="mysql"
         )
 
     @property
@@ -123,13 +191,9 @@ class DatabaseConfig:
         )
 
     @property
-    def user_table_autoincrement_start(self) -> int:
+    def enable_memcached(self) -> bool:
         return CoreConfig.get_config_field(
-            self.__config,
-            "core",
-            "database",
-            "user_table_autoincrement_start",
-            default=10000,
+            self.__config, "core", "database", "enable_memcached", default=True
         )
 
     @property
@@ -138,13 +202,12 @@ class DatabaseConfig:
             self.__config, "core", "database", "memcached_host", default="localhost"
         )
 
-
 class FrontendConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
-    def enable(self) -> int:
+    def enable(self) -> bool:
         return CoreConfig.get_config_field(
             self.__config, "core", "frontend", "enable", default=False
         )
@@ -152,7 +215,7 @@ class FrontendConfig:
     @property
     def port(self) -> int:
         return CoreConfig.get_config_field(
-            self.__config, "core", "frontend", "port", default=8090
+            self.__config, "core", "frontend", "port", default=8080
         )
 
     @property
@@ -162,12 +225,29 @@ class FrontendConfig:
                 self.__config, "core", "frontend", "loglevel", default="info"
             )
         )
-
+    
+    @property
+    def secret(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "frontend", "secret", default=""
+        )
 
 class AllnetConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
+    @property
+    def standalone(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "standalone", default=False
+        )
+    
+    @property
+    def port(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "port", default=80
+        )
+
     @property
     def loglevel(self) -> int:
         return CoreConfig.str_to_loglevel(
@@ -176,12 +256,6 @@ class AllnetConfig:
             )
         )
 
-    @property
-    def port(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "allnet", "port", default=80
-        )
-
     @property
     def allow_online_updates(self) -> int:
         return CoreConfig.get_config_field(
@@ -194,10 +268,23 @@ class AllnetConfig:
             self.__config, "core", "allnet", "update_cfg_folder", default=""
         )
 
-
 class BillingConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
+    
+    @property
+    def standalone(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "billing", "standalone", default=True
+        )
+
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "billing", "loglevel", default="info"
+            )
+        )
 
     @property
     def port(self) -> int:
@@ -223,11 +310,22 @@ class BillingConfig:
             self.__config, "core", "billing", "signing_key", default="cert/billing.key"
         )
 
-
 class AimedbConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
+    @property
+    def enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "enable", default=True
+        )
+
+    @property
+    def listen_address(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "listen_address", default=""
+        )
+
     @property
     def loglevel(self) -> int:
         return CoreConfig.str_to_loglevel(
@@ -248,17 +346,22 @@ class AimedbConfig:
             self.__config, "core", "aimedb", "key", default=""
         )
 
+    @property
+    def id_secret(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "id_secret", default=""
+        )
+
+    @property
+    def id_lifetime_seconds(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "id_lifetime_seconds", default=86400
+        )
 
 class MuchaConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
-    @property
-    def enable(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "mucha", "enable", default=False
-        )
-
     @property
     def loglevel(self) -> int:
         return CoreConfig.str_to_loglevel(
@@ -267,13 +370,6 @@ class MuchaConfig:
             )
         )
 
-    @property
-    def hostname(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "mucha", "hostname", default="localhost"
-        )
-
-
 class CoreConfig(dict):
     def __init__(self) -> None:
         self.server = ServerConfig(self)
@@ -295,6 +391,19 @@ class CoreConfig(dict):
             return logging.DEBUG
         else:
             return logging.INFO
+    
+    @classmethod
+    def loglevel_to_str(cls, level: int) -> str:
+        if level == logging.ERROR:
+            return "error"
+        elif level == logging.WARN:
+            return "warn"
+        elif level == logging.INFO:
+            return "info"
+        elif level == logging.DEBUG:
+            return "debug"
+        else:
+            return "notset"
 
     @classmethod
     def get_config_field(

core/data/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

core/data/alembic/alembic.ini

@@ -0,0 +1,64 @@
+# A generic, single database configuration.
+
+[alembic]
+script_location=.
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to migrations//versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat migrations//versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

core/data/alembic/env.py

@@ -0,0 +1,81 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+from core.data.schema.base import metadata
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    raise Exception('Not implemented or configured!')
+
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    ini_section = config.get_section(config.config_ini_section)
+    overrides = context.get_x_argument(as_dictionary=True)
+    for override in overrides:
+        ini_section[override] = overrides[override]
+
+    connectable = engine_from_config(
+        ini_section,
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            compare_type=True,
+            compare_server_default=True,
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

core/data/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

core/data/alembic/versions/835b862f9bf0_initial_migration.py

@@ -0,0 +1,24 @@
+"""Initial Migration
+
+Revision ID: 835b862f9bf0
+Revises: 
+Create Date: 2024-01-09 13:06:10.787432
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '835b862f9bf0'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

core/data/alembic/versions/d8950c7ce2fc_remove_old_db_mgmt_system.py

@@ -0,0 +1,29 @@
+"""Remove old db mgmt system
+
+Revision ID: d8950c7ce2fc
+Revises: 835b862f9bf0
+Create Date: 2024-01-09 13:43:51.381175
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd8950c7ce2fc'
+down_revision = '835b862f9bf0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_table("schema_versions")
+
+
+def downgrade():
+    op.create_table(
+        "schema_versions",
+        sa.Column("game", sa.String(4), primary_key=True, nullable=False),
+        sa.Column("version", sa.Integer, nullable=False, server_default="1"),
+        mysql_charset="utf8mb4",
+    )

core/data/cache.py

@@ -17,7 +17,7 @@ except ModuleNotFoundError:
 
 def cached(lifetime: int = 10, extra_key: Any = None) -> Callable:
     def _cached(func: Callable) -> Callable:
-        if has_mc:
+        if has_mc and (cfg and cfg.database.enable_memcached):
             hostname = "127.0.0.1"
             if cfg:
                 hostname = cfg.database.memcached_host

core/data/database.py

@@ -1,13 +1,14 @@
 import logging, coloredlogs
-from typing import Optional, Dict, List
+from typing import Optional
 from sqlalchemy.orm import scoped_session, sessionmaker
-from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy import create_engine
 from logging.handlers import TimedRotatingFileHandler
-import importlib, os
+import os
 import secrets, string
 import bcrypt
 from hashlib import sha256
+import alembic.config
+import glob
 
 from core.config import CoreConfig
 from core.data.schema import *
@@ -15,31 +16,47 @@ from core.utils import Utils
 
 
 class Data:
+    engine = None
+    session = None
+    user = None
+    arcade = None
+    card = None
+    base = None
     def __init__(self, cfg: CoreConfig) -> None:
         self.config = cfg
 
         if self.config.database.sha2_password:
             passwd = sha256(self.config.database.password.encode()).digest()
-            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
+            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4"
         else:
-            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
+            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4"
 
-        self.__engine = create_engine(self.__url, pool_recycle=3600)
-        session = sessionmaker(bind=self.__engine, autoflush=True, autocommit=True)
-        self.session = scoped_session(session)
+        if Data.engine is None:
+            Data.engine = create_engine(self.__url, pool_recycle=3600)
+            self.__engine = Data.engine
 
-        self.user = UserData(self.config, self.session)
-        self.arcade = ArcadeData(self.config, self.session)
-        self.card = CardData(self.config, self.session)
-        self.base = BaseData(self.config, self.session)
-        self.current_schema_version = 4
+        if Data.session is None:
+            s = sessionmaker(bind=Data.engine, autoflush=True, autocommit=True)
+            Data.session = scoped_session(s)
+
+        if Data.user is None:
+            Data.user = UserData(self.config, self.session)
+        
+        if Data.arcade is None:
+            Data.arcade = ArcadeData(self.config, self.session)
+        
+        if Data.card is None:
+            Data.card = CardData(self.config, self.session)
+        
+        if Data.base is None:
+            Data.base = BaseData(self.config, self.session)
 
-        log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
         self.logger = logging.getLogger("database")
 
         # Prevent the logger from adding handlers multiple times
-        if not getattr(self.logger, "handler_set", None):
+        if not getattr(self.logger, "handler_set", None):            
+            log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
             fileHandler = TimedRotatingFileHandler(
                 "{0}/{1}.log".format(self.config.server.log_dir, "db"),
                 encoding="utf-8",
@@ -60,276 +77,173 @@ class Data:
             )
             self.logger.handler_set = True  # type: ignore
 
+    def __alembic_cmd(self, command: str, *args: str) -> None:
+        old_dir = os.path.abspath(os.path.curdir)
+        base_dir = os.path.join(os.path.abspath(os.path.curdir), 'core', 'data', 'alembic')
+        alembicArgs = [
+            "-c",
+            os.path.join(base_dir, "alembic.ini"),
+            "-x",
+            f"script_location={base_dir}",
+            "-x",
+            f"sqlalchemy.url={self.__url}",
+            command,
+        ]
+        alembicArgs.extend(args)
+        os.chdir(base_dir)
+        alembic.config.main(argv=alembicArgs)
+        os.chdir(old_dir)
+
     def create_database(self):
         self.logger.info("Creating databases...")
-        try:
-            metadata.create_all(self.__engine.connect())
-        except SQLAlchemyError as e:
-            self.logger.error(f"Failed to create databases! {e}")
-            return
-
-        games = Utils.get_all_titles()
-        for game_dir, game_mod in games.items():
-            try:
-                if hasattr(game_mod, "database") and hasattr(
-                    game_mod, "current_schema_version"
-                ):
-                    game_mod.database(self.config)
-                metadata.create_all(self.__engine.connect())
-
-                self.base.set_schema_ver(
-                    game_mod.current_schema_version, game_mod.game_codes[0]
-                )
-
-            except Exception as e:
-                self.logger.warning(
-                    f"Could not load database schema from {game_dir} - {e}"
-                )
-
-        self.logger.info(f"Setting base_schema_ver to {self.current_schema_version}")
-        self.base.set_schema_ver(self.current_schema_version)
-
-        self.logger.info(
-            f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
-        )
-        self.user.reset_autoincrement(
-            self.config.database.user_table_autoincrement_start
+        metadata.create_all(
+            self.engine,
+            checkfirst=True,
         )
 
-    def recreate_database(self):
-        self.logger.info("Dropping all databases...")
-        self.base.execute("SET FOREIGN_KEY_CHECKS=0")
-        try:
-            metadata.drop_all(self.__engine.connect())
-        except SQLAlchemyError as e:
-            self.logger.error(f"Failed to drop databases! {e}")
-            return
+        for _, mod in Utils.get_all_titles().items():
+            if hasattr(mod, "database"):
+                mod.database(self.config)
+                metadata.create_all(
+                    self.engine,
+                    checkfirst=True,
+                )
 
-        for root, dirs, files in os.walk("./titles"):
-            for dir in dirs:
-                if not dir.startswith("__"):
-                    try:
-                        mod = importlib.import_module(f"titles.{dir}")
+        # Stamp the end revision as if alembic had created it, so it can take off after this.
+        self.__alembic_cmd(
+            "stamp",
+            "head",
+        )
 
-                        try:
-                            if hasattr(mod, "database"):
-                                mod.database(self.config)
-                            metadata.drop_all(self.__engine.connect())
+    def schema_upgrade(self, ver: str = None):
+        self.__alembic_cmd(
+            "upgrade",
+            "head" if not ver else ver,
+        )
 
-                        except Exception as e:
-                            self.logger.warning(
-                                f"Could not load database schema from {dir} - {e}"
-                            )
+    def schema_downgrade(self, ver: str):
+        self.__alembic_cmd(
+            "downgrade",
+            ver,
+        )
 
-                    except ImportError as e:
-                        self.logger.warning(
-                            f"Failed to load database schema dir {dir} - {e}"
-                        )
-            break
-
-        self.base.execute("SET FOREIGN_KEY_CHECKS=1")
-
-        self.create_database()
-
-    def migrate_database(self, game: str, version: Optional[int], action: str) -> None:
-        old_ver = self.base.get_schema_ver(game)
-        sql = ""
-        if version is None:
-            if not game == "CORE":
-                titles = Utils.get_all_titles()
-
-                for folder, mod in titles.items():
-                    if not mod.game_codes[0] == game:
-                        continue
-
-                    if hasattr(mod, "current_schema_version"):
-                        version = mod.current_schema_version
-
-                    else:
-                        self.logger.warn(
-                            f"current_schema_version not found for {folder}"
-                        )
-
-            else:
-                version = self.current_schema_version
-
-        if version is None:
-            self.logger.warn(
-                f"Could not determine latest version for {game}, please specify --version"
-            )
-
-        if old_ver is None:
-            self.logger.error(
-                f"Schema for game {game} does not exist, did you run the creation script?"
-            )
-            return
-
-        if old_ver == version:
-            self.logger.info(
-                f"Schema for game {game} is already version {old_ver}, nothing to do"
-            )
-            return
-
-        if action == "upgrade":
-            for x in range(old_ver, version):
-                if not os.path.exists(
-                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql"
-                ):
-                    self.logger.error(
-                        f"Could not find {action} script {game.upper()}_{x + 1}_{action}.sql in core/data/schema/versions folder"
-                    )
-                    return
-
-                with open(
-                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql",
-                    "r",
-                    encoding="utf-8",
-                ) as f:
-                    sql = f.read()
-
-                result = self.base.execute(sql)
-                if result is None:
-                    self.logger.error("Error execuing sql script!")
-                    return None
-
-        else:
-            for x in range(old_ver, version, -1):
-                if not os.path.exists(
-                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql"
-                ):
-                    self.logger.error(
-                        f"Could not find {action} script {game.upper()}_{x - 1}_{action}.sql in core/data/schema/versions folder"
-                    )
-                    return
-
-                with open(
-                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql",
-                    "r",
-                    encoding="utf-8",
-                ) as f:
-                    sql = f.read()
-
-                result = self.base.execute(sql)
-                if result is None:
-                    self.logger.error("Error execuing sql script!")
-                    return None
-
-        result = self.base.set_schema_ver(version, game)
-        if result is None:
-            self.logger.error("Error setting version in schema_version table!")
-            return None
-
-        self.logger.info(f"Successfully migrated {game} to schema version {version}")
-
-    def create_owner(self, email: Optional[str] = None) -> None:
+    async def create_owner(self, email: Optional[str] = None, code: Optional[str] = "00000000000000000000") -> None:
         pw = "".join(
             secrets.choice(string.ascii_letters + string.digits) for i in range(20)
         )
         hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt())
 
-        user_id = self.user.create_user(email=email, permission=255, password=hash)
+        user_id = await self.user.create_user(username="sysowner", email=email, password=hash.decode(), permission=255)
         if user_id is None:
             self.logger.error(f"Failed to create owner with email {email}")
             return
 
-        card_id = self.card.create_card(user_id, "00000000000000000000")
+        card_id = await self.card.create_card(user_id, code)
         if card_id is None:
             self.logger.error(f"Failed to create card for owner with id {user_id}")
             return
 
-        self.logger.warn(
-            f"Successfully created owner with email {email}, access code 00000000000000000000, and password {pw} Make sure to change this password and assign a real card ASAP!"
+        self.logger.warning(
+            f"Successfully created owner with email {email}, access code {code}, and password {pw} Make sure to change this password and assign a real card ASAP!"
+        )
+    
+    async def migrate(self) -> None:
+        exist = await self.base.execute("SELECT * FROM alembic_version")
+        if exist is not None:
+            self.logger.warn("No need to migrate as you have already migrated to alembic. If you are trying to upgrade the schema, use `upgrade` instead!")
+            return
+        
+        self.logger.info("Upgrading to latest with legacy system")
+        if not await self.legacy_upgrade():
+            self.logger.warn("No need to migrate as you have already deleted the old schema_versions system. If you are trying to upgrade the schema, use `upgrade` instead!")
+            return
+        self.logger.info("Done")
+        
+        self.logger.info("Stamp with initial revision")
+        self.__alembic_cmd(
+            "stamp",
+            "835b862f9bf0",
         )
 
-    def migrate_card(self, old_ac: str, new_ac: str, should_force: bool) -> None:
-        if old_ac == new_ac:
-            self.logger.error("Both access codes are the same!")
-            return
-
-        new_card = self.card.get_card_by_access_code(new_ac)
-        if new_card is None:
-            self.card.update_access_code(old_ac, new_ac)
-            return
-
-        if not should_force:
-            self.logger.warn(
-                f"Card already exists for access code {new_ac} (id {new_card['id']}). If you wish to continue, rerun with the '--force' flag."
-                f" All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
-            )
-            return
-
-        self.logger.info(
-            f"All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
+        self.logger.info("Upgrade")
+        self.__alembic_cmd(
+            "upgrade",
+            "head",
         )
-        self.card.delete_card(new_card["id"])
-        self.card.update_access_code(old_ac, new_ac)
+    
+    async def legacy_upgrade(self) -> bool:
+        vers = await self.base.execute("SELECT * FROM schema_versions")
+        if vers is None:
+            self.logger.warn("Cannot legacy upgrade, schema_versions table unavailable!")
+            return False
+        
+        db_vers = {}
+        vers_list = vers.fetchall()
+        for x in vers_list:
+            db_vers[x['game']] = x['version']
+        
+        core_now_ver = int(db_vers['CORE']) + 1
+        while os.path.exists(f"core/data/schema/versions/CORE_{core_now_ver}_upgrade.sql"):
+            with open(f"core/data/schema/versions/CORE_{core_now_ver}_upgrade.sql", "r") as f:
+                result = await self.base.execute(f.read())
+                
+                if result is None:
+                    self.logger.error(f"Invalid upgrade script CORE_{core_now_ver}_upgrade.sql")
+                    break
 
-        hanging_user = self.user.get_user(new_card["user"])
-        if hanging_user["password"] is None:
-            self.logger.info(f"Delete hanging user {hanging_user['id']}")
-            self.user.delete_user(hanging_user["id"])
+                result = await self.base.execute(f"UPDATE schema_versions SET version = {core_now_ver} WHERE game = 'CORE'")
+                if result is None:
+                    self.logger.error(f"Failed to update schema version for CORE to {core_now_ver}")
+                    break
+            
+            self.logger.info(f"Upgrade CORE to version {core_now_ver}")
+            core_now_ver += 1
+        
+        for _, mod in Utils.get_all_titles().items():
+            game_codes = getattr(mod, "game_codes", [])
+            for game in game_codes:
+                if game not in db_vers:
+                    self.logger.warn(f"{game} does not have an antry in schema_versions, skipping")
+                    continue
 
-    def delete_hanging_users(self) -> None:
-        """
-        Finds and deletes users that have not registered for the webui that have no cards assocated with them.
-        """
-        unreg_users = self.user.get_unregistered_users()
-        if unreg_users is None:
-            self.logger.error("Error occoured finding unregistered users")
+                now_ver = int(db_vers[game]) + 1
+                while os.path.exists(f"core/data/schema/versions/{game}_{now_ver}_upgrade.sql"):
+                    with open(f"core/data/schema/versions/{game}_{now_ver}_upgrade.sql", "r") as f:
+                        result = await self.base.execute(f.read())
+                        
+                        if result is None:
+                            self.logger.error(f"Invalid upgrade script {game}_{now_ver}_upgrade.sql")
+                            break
 
-        for user in unreg_users:
-            cards = self.card.get_user_cards(user["id"])
-            if cards is None:
-                self.logger.error(f"Error getting cards for user {user['id']}")
-                continue
+                        result = await self.base.execute(f"UPDATE schema_versions SET version = {now_ver} WHERE game = '{game}'")
+                        if result is None:
+                            self.logger.error(f"Failed to update schema version for {game} to {now_ver}")
+                            break
 
-            if not cards:
-                self.logger.info(f"Delete hanging user {user['id']}")
-                self.user.delete_user(user["id"])
+                    self.logger.info(f"Upgrade {game} to version {now_ver}")
+                    now_ver += 1
+        
+        return True
 
-    def autoupgrade(self) -> None:
-        all_game_versions = self.base.get_all_schema_vers()
-        if all_game_versions is None:
-            self.logger.warn("Failed to get schema versions")
+    async def create_revision(self, message: str) -> None:
+        if not message:
+            self.logger.info("Message is required for create-revision")
             return
+        
+        self.__alembic_cmd(
+            "revision",
+            "-m",
+            message,
+        )
 
-        all_games = Utils.get_all_titles()
-        all_games_list: Dict[str, int] = {}
-        for _, mod in all_games.items():
-            if hasattr(mod, "current_schema_version"):
-                all_games_list[mod.game_codes[0]] = mod.current_schema_version
-
-        for x in all_game_versions:
-            failed = False
-            game = x["game"].upper()
-            update_ver = int(x["version"])
-            latest_ver = all_games_list.get(game, 1)
-            if game == "CORE":
-                latest_ver = self.current_schema_version
-
-            if update_ver == latest_ver:
-                self.logger.info(f"{game} is already latest version")
-                continue
-
-            for y in range(update_ver + 1, latest_ver + 1):
-                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
-                    with open(
-                        f"core/data/schema/versions/{game}_{y}_upgrade.sql",
-                        "r",
-                        encoding="utf-8",
-                    ) as f:
-                        sql = f.read()
-
-                    result = self.base.execute(sql)
-                    if result is None:
-                        self.logger.error(
-                            f"Error execuing sql script for game {game} v{y}!"
-                        )
-                        failed = True
-                        break
-                else:
-                    self.logger.warning(f"Could not find script {game}_{y}_upgrade.sql")
-                    failed = True
-
-            if not failed:
-                self.base.set_schema_ver(latest_ver, game)
+    async def create_revision_auto(self, message: str) -> None:
+        if not message:
+            self.logger.info("Message is required for create-revision")
+            return
+        
+        self.__alembic_cmd(
+            "revision",
+            "--autogenerate",
+            "-m",
+            message,
+        )

core/data/schema/arcade.py

@@ -1,9 +1,10 @@
-from typing import Optional, Dict
-from sqlalchemy import Table, Column
+from typing import Optional, Dict, List
+from sqlalchemy import Table, Column, and_, or_
 from sqlalchemy.sql.schema import ForeignKey, PrimaryKeyConstraint
-from sqlalchemy.types import Integer, String, Boolean
+from sqlalchemy.types import Integer, String, Boolean, JSON
 from sqlalchemy.sql import func, select
 from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.engine import Row
 import re
 
 from core.data.schema.base import BaseData, metadata
@@ -21,6 +22,7 @@ arcade = Table(
     Column("city", String(255)),
     Column("region_id", Integer),
     Column("timezone", String(255)),
+    Column("ip", String(39)),
     mysql_charset="utf8mb4",
 )
 
@@ -39,7 +41,9 @@ machine = Table(
     Column("country", String(3)),  # overwrites if not null
     Column("timezone", String(255)),
     Column("ota_enable", Boolean),
+    Column("memo", String(255)),
     Column("is_cab", Boolean),
+    Column("data", JSON),
     mysql_charset="utf8mb4",
 )
 
@@ -65,7 +69,7 @@ arcade_owner = Table(
 
 
 class ArcadeData(BaseData):
-    def get_machine(self, serial: str = None, id: int = None) -> Optional[Dict]:
+    async def get_machine(self, serial: str = None, id: int = None) -> Optional[Row]:
         if serial is not None:
             serial = serial.replace("-", "")
             if len(serial) == 11:
@@ -85,12 +89,12 @@ class ArcadeData(BaseData):
             self.logger.error(f"{__name__ }: Need either serial or ID to look up!")
             return None
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_machine(
+    async def put_machine(
         self,
         arcade_id: int,
         serial: str = "",
@@ -106,13 +110,13 @@ class ArcadeData(BaseData):
             arcade=arcade_id, keychip=serial, board=board, game=game, is_cab=is_cab
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def set_machine_serial(self, machine_id: int, serial: str) -> None:
-        result = self.execute(
+    async def set_machine_serial(self, machine_id: int, serial: str) -> None:
+        result = await self.execute(
             machine.update(machine.c.id == machine_id).values(keychip=serial)
         )
         if result is None:
@@ -121,8 +125,8 @@ class ArcadeData(BaseData):
             )
         return result.lastrowid
 
-    def set_machine_boardid(self, machine_id: int, boardid: str) -> None:
-        result = self.execute(
+    async def set_machine_boardid(self, machine_id: int, boardid: str) -> None:
+        result = await self.execute(
             machine.update(machine.c.id == machine_id).values(board=boardid)
         )
         if result is None:
@@ -130,14 +134,21 @@ class ArcadeData(BaseData):
                 f"Failed to update board id for machine {machine_id} -> {boardid}"
             )
 
-    def get_arcade(self, id: int) -> Optional[Dict]:
+    async def get_arcade(self, id: int) -> Optional[Row]:
         sql = arcade.select(arcade.c.id == id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
+    
+    async def get_arcade_machines(self, id: int) -> Optional[List[Row]]:
+        sql = machine.select(machine.c.arcade == id)
+        result = await self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
 
-    def put_arcade(
+    async def put_arcade(
         self,
         name: str,
         nickname: str = None,
@@ -160,60 +171,62 @@ class ArcadeData(BaseData):
             regional_id=regional_id,
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_arcade_owners(self, arcade_id: int) -> Optional[Dict]:
+    async def get_arcades_managed_by_user(self, user_id: int) -> Optional[List[Row]]:
+        sql = select(arcade).join(arcade_owner, arcade_owner.c.arcade == arcade.c.id).where(arcade_owner.c.user == user_id)
+        result = await self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchall()
+    
+    async def get_manager_permissions(self, user_id: int, arcade_id: int) -> Optional[int]:
+        sql = select(arcade_owner.c.permissions).where(and_(arcade_owner.c.user == user_id, arcade_owner.c.arcade == arcade_id))
+        result = await self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchone()
+
+    async def get_arcade_owners(self, arcade_id: int) -> Optional[Row]:
         sql = select(arcade_owner).where(arcade_owner.c.arcade == arcade_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
+    async def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
         sql = insert(arcade_owner).values(arcade=arcade_id, user=user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def format_serial(
+    async def format_serial(
         self, platform_code: str, platform_rev: int, serial_num: int, append: int = 4152
     ) -> str:
         return f"{platform_code}{platform_rev:02d}A{serial_num:04d}{append:04d}"  # 0x41 = A, 0x52 = R
 
     def validate_keychip_format(self, serial: str) -> bool:
-        serial = serial.replace("-", "")
-        if len(serial) != 11 or len(serial) != 15:
-            self.logger.error(
-                f"Serial validate failed: Incorrect length for {serial} (len {len(serial)})"
-            )
+        if re.fullmatch(r"^A[0-9]{2}[E|X][-]?[0-9]{2}[A-HJ-NP-Z][0-9]{4}([0-9]{4})?$", serial) is None:
             return False
-
-        platform_code = serial[:4]
-        platform_rev = serial[4:6]
-        const_a = serial[6]
-        num = serial[7:11]
-        append = serial[11:15]
-
-        if re.match("A[7|6]\d[E|X][0|1][0|1|2]A\d{4,8}", serial) is None:
-            self.logger.error(f"Serial validate failed: {serial} failed regex")
-            return False
-
-        if len(append) != 0 or len(append) != 4:
-            self.logger.error(
-                f"Serial validate failed: {serial} had malformed append {append}"
-            )
-            return False
-
-        if len(num) != 4:
-            self.logger.error(
-                f"Serial validate failed: {serial} had malformed number {num}"
-            )
-            return False
-
+        
         return True
+
+    async def get_arcade_by_name(self, name: str) -> Optional[List[Row]]:
+        sql = arcade.select(or_(arcade.c.name.like(f"%{name}%"), arcade.c.nickname.like(f"%{name}%")))
+        result = await self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    async def get_arcades_by_ip(self, ip: str) -> Optional[List[Row]]:
+        sql = arcade.select().where(arcade.c.ip == ip)
+        result = await self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()

core/data/schema/base.py

@@ -15,14 +15,6 @@ from core.config import CoreConfig
 
 metadata = MetaData()
 
-schema_ver = Table(
-    "schema_versions",
-    metadata,
-    Column("game", String(4), primary_key=True, nullable=False),
-    Column("version", Integer, nullable=False, server_default="1"),
-    mysql_charset="utf8mb4",
-)
-
 event_log = Table(
     "event_log",
     metadata,
@@ -43,11 +35,11 @@ class BaseData:
         self.conn = conn
         self.logger = logging.getLogger("database")
 
-    def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
+    async def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
         res = None
 
         try:
-            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())}")
+            self.logger.debug(f"SQL Execute: {''.join(str(sql).splitlines())}")
             res = self.conn.execute(text(sql), opts)
 
         except SQLAlchemyError as e:
@@ -58,7 +50,7 @@ class BaseData:
             self.logger.error(f"UnicodeEncodeError error {e}")
             return None
 
-        except:
+        except Exception:
             try:
                 res = self.conn.execute(sql, opts)
 
@@ -70,7 +62,7 @@ class BaseData:
                 self.logger.error(f"UnicodeEncodeError error {e}")
                 return None
 
-            except:
+            except Exception:
                 self.logger.error(f"Unknown error")
                 raise
 
@@ -82,40 +74,7 @@ class BaseData:
         """
         return randrange(10000, 9999999)
 
-    def get_all_schema_vers(self) -> Optional[List[Row]]:
-        sql = select(schema_ver)
-
-        result = self.execute(sql)
-        if result is None:
-            return None
-        return result.fetchall()
-
-    def get_schema_ver(self, game: str) -> Optional[int]:
-        sql = select(schema_ver).where(schema_ver.c.game == game)
-
-        result = self.execute(sql)
-        if result is None:
-            return None
-
-        row = result.fetchone()
-        if row is None:
-            return None
-
-        return row["version"]
-
-    def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = insert(schema_ver).values(game=game, version=ver)
-        conflict = sql.on_duplicate_key_update(version=ver)
-
-        result = self.execute(conflict)
-        if result is None:
-            self.logger.error(
-                f"Failed to update schema version for game {game} (v{ver})"
-            )
-            return None
-        return result.lastrowid
-
-    def log_event(
+    async def log_event(
         self, system: str, type: str, severity: int, message: str, details: Dict = {}
     ) -> Optional[int]:
         sql = event_log.insert().values(
@@ -125,7 +84,7 @@ class BaseData:
             message=message,
             details=json.dumps(details),
         )
-        result = self.execute(sql)
+        result = await self.execute(sql)
 
         if result is None:
             self.logger.error(
@@ -135,9 +94,9 @@ class BaseData:
 
         return result.lastrowid
 
-    def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
+    async def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
         sql = event_log.select().limit(entries).all()
-        result = self.execute(sql)
+        result = await self.execute(sql)
 
         if result is None:
             return None
@@ -145,6 +104,8 @@ class BaseData:
 
     def fix_bools(self, data: Dict) -> Dict:
         for k, v in data.items():
+            if k == "userName" or k == "teamName":
+                continue
             if type(v) == str and v.lower() == "true":
                 data[k] = True
             elif type(v) == str and v.lower() == "false":

core/data/schema/card.py

@@ -27,70 +27,101 @@ aime_card = Table(
 
 
 class CardData(BaseData):
-    def get_card_by_access_code(self, access_code: str) -> Optional[Row]:
+    async def get_card_by_access_code(self, access_code: str) -> Optional[Row]:
         sql = aime_card.select(aime_card.c.access_code == access_code)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def get_card_by_id(self, card_id: int) -> Optional[Row]:
+    async def get_card_by_id(self, card_id: int) -> Optional[Row]:
         sql = aime_card.select(aime_card.c.id == card_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def update_access_code(self, old_ac: str, new_ac: str) -> None:
+    async def update_access_code(self, old_ac: str, new_ac: str) -> None:
         sql = aime_card.update(aime_card.c.access_code == old_ac).values(
             access_code=new_ac
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(
                 f"Failed to change card access code from {old_ac} to {new_ac}"
             )
 
-    def get_user_id_from_card(self, access_code: str) -> Optional[int]:
+    async def get_user_id_from_card(self, access_code: str) -> Optional[int]:
         """
         Given a 20 digit access code as a string, get the user id associated with that card
         """
-        card = self.get_card_by_access_code(access_code)
+        card = await self.get_card_by_access_code(access_code)
         if card is None:
             return None
 
         return int(card["user"])
 
-    def delete_card(self, card_id: int) -> None:
+    async def get_card_banned(self, access_code: str) -> Optional[bool]:
+        """
+        Given a 20 digit access code as a string, check if the card is banned
+        """
+        card = await self.get_card_by_access_code(access_code)
+        if card is None:
+            return None
+        if card["is_banned"]:
+            return True
+        return False
+    
+    async def get_card_locked(self, access_code: str) -> Optional[bool]:
+        """
+        Given a 20 digit access code as a string, check if the card is locked
+        """
+        card = await self.get_card_by_access_code(access_code)
+        if card is None:
+            return None
+        if card["is_locked"]:
+            return True
+        return False
+
+    async def delete_card(self, card_id: int) -> None:
         sql = aime_card.delete(aime_card.c.id == card_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(f"Failed to delete card with id {card_id}")
 
-    def get_user_cards(self, aime_id: int) -> Optional[List[Row]]:
+    async def get_user_cards(self, aime_id: int) -> Optional[List[Row]]:
         """
         Returns all cards owned by a user
         """
         sql = aime_card.select(aime_card.c.user == aime_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def create_card(self, user_id: int, access_code: str) -> Optional[int]:
+    async def create_card(self, user_id: int, access_code: str) -> Optional[int]:
         """
         Given a aime_user id and a 20 digit access code as a string, create a card and return the ID if successful
         """
         sql = aime_card.insert().values(user=user_id, access_code=access_code)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
+    async def update_card_last_login(self, access_code: str) -> None:
+        sql = aime_card.update(aime_card.c.access_code == access_code).values(
+            last_login_date=func.now()
+        )
+        
+        result = await self.execute(sql)
+        if result is None:
+            self.logger.warn(f"Failed to update last login time for {access_code}")
+    
     def to_access_code(self, luid: str) -> str:
         """
         Given a felica cards internal 16 hex character luid, convert it to a 0-padded 20 digit access code as a string

core/data/schema/user.py

@@ -1,4 +1,3 @@
-from enum import Enum
 from typing import Optional, List
 from sqlalchemy import Table, Column
 from sqlalchemy.types import Integer, String, TIMESTAMP
@@ -24,15 +23,8 @@ aime_user = Table(
     mysql_charset="utf8mb4",
 )
 
-
-class PermissionBits(Enum):
-    PermUser = 1
-    PermMod = 2
-    PermSysAdmin = 4
-
-
 class UserData(BaseData):
-    def create_user(
+    async def create_user(
         self,
         id: int = None,
         username: str = None,
@@ -60,47 +52,71 @@ class UserData(BaseData):
             username=username, email=email, password=password, permissions=permission
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_user(self, user_id: int) -> Optional[Row]:
+    async def get_user(self, user_id: int) -> Optional[Row]:
         sql = select(aime_user).where(aime_user.c.id == user_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchone()
 
-    def check_password(self, user_id: int, passwd: bytes = None) -> bool:
-        usr = self.get_user(user_id)
+    async def check_password(self, user_id: int, passwd: bytes = None) -> bool:
+        usr = await self.get_user(user_id)
         if usr is None:
             return False
 
         if usr["password"] is None:
             return False
+        
+        if passwd is None or not passwd:
+            return False
 
         return bcrypt.checkpw(passwd, usr["password"].encode())
 
-    def reset_autoincrement(self, ai_value: int) -> None:
-        # ALTER TABLE isn't in sqlalchemy so we do this the ugly way
-        sql = f"ALTER TABLE aime_user AUTO_INCREMENT={ai_value}"
-        self.execute(sql)
-
-    def delete_user(self, user_id: int) -> None:
+    async def delete_user(self, user_id: int) -> None:
         sql = aime_user.delete(aime_user.c.id == user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(f"Failed to delete user with id {user_id}")
 
-    def get_unregistered_users(self) -> List[Row]:
+    async def get_unregistered_users(self) -> List[Row]:
         """
         Returns a list of users who have not registered with the webui. They may or may not have cards.
         """
         sql = select(aime_user).where(aime_user.c.password == None)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
+
+    async def find_user_by_email(self, email: str) -> Row:
+        sql = select(aime_user).where(aime_user.c.email == email)
+        result = await self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchone()
+
+    async def find_user_by_username(self, username: str) -> List[Row]:
+        sql = aime_user.select(aime_user.c.username.like(f"%{username}%"))
+        result = await self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchall()
+
+    async def change_password(self, user_id: int, new_passwd: str) -> bool:
+        sql = aime_user.update(aime_user.c.id == user_id).values(password = new_passwd)
+
+        result = await self.execute(sql)
+        return result is not None
+
+    async def change_username(self, user_id: int, new_name: str) -> bool:
+        sql = aime_user.update(aime_user.c.id == user_id).values(username = new_name)
+
+        result = await self.execute(sql)
+        return result is not None

core/data/schema/versions/CORE_4_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE machine DROP COLUMN memo;
+ALTER TABLE machine DROP COLUMN is_blacklisted;
+ALTER TABLE machine DROP COLUMN `data`;

core/data/schema/versions/CORE_5_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE arcade DROP COLUMN 'ip';

core/data/schema/versions/CORE_5_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE machine ADD memo varchar(255) NULL;
+ALTER TABLE machine ADD is_blacklisted tinyint(1) NULL;
+ALTER TABLE machine ADD `data` longtext NULL;

core/data/schema/versions/CORE_6_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE arcade ADD ip varchar(39) NULL;

core/data/schema/versions/SBZV_5_rollback.sql

@@ -0,0 +1,2 @@
+ALTER TABLE diva_profile
+    DROP skn_eqp;

core/data/schema/versions/SBZV_6_upgrade.sql

@@ -0,0 +1,2 @@
+ALTER TABLE diva_profile
+    ADD skn_eqp INT NOT NULL DEFAULT 0;

core/data/schema/versions/SDBT_3_rollback.sql

@@ -0,0 +1,30 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    DROP COLUMN regionId,
+    DROP COLUMN machineType;
+
+ALTER TABLE chuni_static_events
+    DROP COLUMN startDate;
+
+ALTER TABLE chuni_profile_data
+    DROP COLUMN rankUpChallengeResults;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN presetId id INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD PRIMARY KEY(id);
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_uk UNIQUE(id, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY(presetId)
+    REFERENCES chuni_static_login_bonus_preset(id) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_4_rollback.sql

@@ -0,0 +1,12 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+  CHANGE COLUMN isClear isClear TINYINT(1) NULL DEFAULT NULL;
+
+ALTER TABLE chuni_score_best
+  CHANGE COLUMN isSuccess isSuccess TINYINT(1) NULL DEFAULT NULL ;
+
+ALTER TABLE chuni_score_playlog
+  DROP COLUMN ticketId;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_4_upgrade.sql

@@ -0,0 +1,29 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    ADD COLUMN regionId INT,
+    ADD COLUMN machineType INT;
+
+ALTER TABLE chuni_static_events
+    ADD COLUMN startDate TIMESTAMP NOT NULL DEFAULT current_timestamp();
+
+ALTER TABLE chuni_profile_data
+    ADD COLUMN rankUpChallengeResults JSON;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN id presetId INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP INDEX chuni_static_login_bonus_preset_uk;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_pk PRIMARY KEY (presetId, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY (presetId, version)
+    REFERENCES chuni_static_login_bonus_preset(presetId, version) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_5_upgrade.sql

@@ -0,0 +1,12 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+  CHANGE COLUMN isClear isClear TINYINT(6) NULL DEFAULT NULL;
+
+ALTER TABLE chuni_score_best
+  CHANGE COLUMN isSuccess isSuccess INT(11) NULL DEFAULT NULL ;
+
+ALTER TABLE chuni_score_playlog
+  ADD COLUMN ticketId INT(11) NULL AFTER machineType;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDDT_4_rollback.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ongeki_static_events
+DROP COLUMN startDate;

core/data/schema/versions/SDDT_5_rollback.sql

@@ -0,0 +1,22 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+ALTER TABLE ongeki_user_event_point DROP COLUMN version;
+ALTER TABLE ongeki_user_event_point DROP COLUMN `rank`;
+ALTER TABLE ongeki_user_event_point DROP COLUMN `type`;
+ALTER TABLE ongeki_user_event_point DROP COLUMN date;
+
+ALTER TABLE ongeki_user_tech_event DROP COLUMN version;
+
+ALTER TABLE ongeki_user_mission_point DROP COLUMN version;
+
+ALTER TABLE ongeki_static_events DROP COLUMN endDate;
+
+DROP TABLE ongeki_tech_event_ranking;
+DROP TABLE ongeki_static_music_ranking_list;
+DROP TABLE ongeki_static_rewards;
+DROP TABLE ongeki_static_present_list;
+DROP TABLE ongeki_static_tech_music;
+DROP TABLE ongeki_static_client_testmode;
+DROP TABLE ongeki_static_game_point;
+
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDDT_5_upgrade.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ongeki_static_events
+ADD COLUMN startDate TIMESTAMP NOT NULL DEFAULT current_timestamp();

core/data/schema/versions/SDDT_6_upgrade.sql

@@ -0,0 +1,98 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+ALTER TABLE ongeki_user_event_point ADD COLUMN version INTEGER NOT NULL;
+ALTER TABLE ongeki_user_event_point ADD COLUMN `rank` INTEGER;
+ALTER TABLE ongeki_user_event_point ADD COLUMN `type` INTEGER NOT NULL;
+ALTER TABLE ongeki_user_event_point ADD COLUMN date VARCHAR(25);
+
+ALTER TABLE ongeki_user_tech_event ADD COLUMN version INTEGER NOT NULL;
+
+ALTER TABLE ongeki_user_mission_point ADD COLUMN version INTEGER NOT NULL;
+
+ALTER TABLE ongeki_static_events ADD COLUMN endDate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
+
+CREATE TABLE ongeki_tech_event_ranking (
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    user INT NOT NULL,
+    version INT NOT NULL,
+    date VARCHAR(25),
+    eventId INT NOT NULL,
+    `rank` INT,
+    totalPlatinumScore INT NOT NULL,
+    totalTechScore INT NOT NULL,
+    UNIQUE KEY ongeki_tech_event_ranking_uk (user, eventId),
+    CONSTRAINT ongeki_tech_event_ranking_ibfk1 FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE TABLE ongeki_static_music_ranking_list (
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    version INT NOT NULL,
+    musicId INT NOT NULL,
+    point INT NOT NULL,
+    userName VARCHAR(255),
+    UNIQUE KEY ongeki_static_music_ranking_list_uk (version, musicId)
+);
+
+CREATE TABLE ongeki_static_rewards (
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    version INT NOT NULL,
+    rewardId INT NOT NULL,
+    rewardName VARCHAR(255) NOT NULL,
+    itemKind INT NOT NULL,
+    itemId INT NOT NULL,
+    UNIQUE KEY ongeki_tech_event_ranking_uk (version, rewardId)
+);
+
+CREATE TABLE ongeki_static_present_list (
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    version INT NOT NULL,
+    presentId INT NOT NULL,
+    presentName VARCHAR(255) NOT NULL,
+    rewardId INT NOT NULL,
+    stock INT NOT NULL,
+    message VARCHAR(255),
+    startDate VARCHAR(25) NOT NULL,
+    endDate VARCHAR(25) NOT NULL,
+    UNIQUE KEY ongeki_static_present_list_uk (version, presentId, rewardId)
+);
+
+CREATE TABLE ongeki_static_tech_music (
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    version INT NOT NULL,
+    eventId INT NOT NULL,
+    musicId INT NOT NULL,
+    level INT NOT NULL,
+    UNIQUE KEY ongeki_static_tech_music_uk (version, musicId, eventId)
+);
+
+CREATE TABLE ongeki_static_client_testmode (
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    regionId INT NOT NULL,
+    placeId INT NOT NULL,
+    clientId VARCHAR(11) NOT NULL,
+    updateDate TIMESTAMP NOT NULL,
+    isDelivery BOOLEAN NOT NULL,
+    groupId INT NOT NULL,
+    groupRole INT NOT NULL,
+    continueMode INT NOT NULL,
+    selectMusicTime INT NOT NULL,
+    advertiseVolume INT NOT NULL,
+    eventMode INT NOT NULL,
+    eventMusicNum INT NOT NULL,
+    patternGp INT NOT NULL,
+    limitGp INT NOT NULL,
+    maxLeverMovable INT NOT NULL,
+    minLeverMovable INT NOT NULL,
+    UNIQUE KEY ongeki_static_client_testmode_uk (clientId)
+);
+
+CREATE TABLE ongeki_static_game_point (
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    `type` INT NOT NULL,
+    cost INT NOT NULL,
+    startDate VARCHAR(25) NOT NULL DEFAULT "2000-01-01 05:00:00.0",
+    endDate VARCHAR(25) NOT NULL DEFAULT "2099-01-01 05:00:00.0",
+    UNIQUE KEY ongeki_static_game_point_uk (`type`)
+);
+
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDEZ_4_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP DEFAULT "2018-01-01 00:00:00.0",
+    CHANGE COLUMN endDate endDate TIMESTAMP DEFAULT "2038-01-01 00:00:00.0";

core/data/schema/versions/SDEZ_5_rollback.sql

@@ -0,0 +1,78 @@
+DELETE FROM mai2_static_event WHERE version < 13;
+UPDATE mai2_static_event SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_music WHERE version < 13;
+UPDATE mai2_static_music SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_ticket WHERE version < 13;
+UPDATE mai2_static_ticket SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_cards WHERE version < 13;
+UPDATE mai2_static_cards SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_detail WHERE version < 13;
+UPDATE mai2_profile_detail SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_extend WHERE version < 13;
+UPDATE mai2_profile_extend SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_option WHERE version < 13;
+UPDATE mai2_profile_option SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_ghost WHERE version < 13;
+UPDATE mai2_profile_ghost SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_rating WHERE version < 13;
+UPDATE mai2_profile_rating SET version = version - 13 WHERE version >= 13;
+
+DROP TABLE maimai_score_best;
+DROP TABLE maimai_playlog;
+DROP TABLE maimai_profile_detail;
+DROP TABLE maimai_profile_option;
+DROP TABLE maimai_profile_web_option;
+DROP TABLE maimai_profile_grade_status;
+
+ALTER TABLE mai2_item_character DROP COLUMN point;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NOT NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NOT NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NOT NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NOT NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NOT NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN `rank` int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NOT NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;

core/data/schema/versions/SDEZ_5_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CHANGE COLUMN endDate endDate TIMESTAMP NOT NULL;

core/data/schema/versions/SDEZ_6_rollback.sql

@@ -0,0 +1 @@
+DROP TABLE aime.mai2_profile_consec_logins;

core/data/schema/versions/SDEZ_6_upgrade.sql

@@ -0,0 +1,62 @@
+UPDATE mai2_static_event SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_music SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_ticket SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_cards SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_detail  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_extend  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_option  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_ghost  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_rating  SET version = version + 13 WHERE version < 1000;
+
+ALTER TABLE mai2_item_character ADD point int(11) NULL;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN `rank` int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;

core/data/schema/versions/SDEZ_7_rollback.sql

@@ -0,0 +1,10 @@
+ALTER TABLE mai2_profile_detail
+    DROP COLUMN mapStock;
+
+ALTER TABLE mai2_profile_extend
+    DROP COLUMN selectResultScoreViewType;
+
+ALTER TABLE mai2_profile_option
+    DROP COLUMN outFrameType,
+    DROP COLUMN touchVolume,
+    DROP COLUMN breakSlideVolume;

core/data/schema/versions/SDEZ_7_upgrade.sql

@@ -0,0 +1,9 @@
+CREATE TABLE `mai2_profile_consec_logins` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `user` int(11) NOT NULL,
+  `version` int(11) NOT NULL,
+  `logins` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `mai2_profile_consec_logins_uk` (`user`,`version`),
+  CONSTRAINT `mai2_profile_consec_logins_ibfk_1` FOREIGN KEY (`user`) REFERENCES `aime_user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

core/data/schema/versions/SDEZ_8_upgrade.sql

@@ -0,0 +1,10 @@
+ALTER TABLE mai2_profile_detail
+    ADD mapStock INT NULL AFTER playCount;
+
+ALTER TABLE mai2_profile_extend
+    ADD selectResultScoreViewType INT NULL AFTER selectResultDetails;
+
+ALTER TABLE mai2_profile_option
+    ADD outFrameType INT NULL AFTER dispCenter,
+    ADD touchVolume INT NULL AFTER slideVolume,
+    ADD breakSlideVolume INT NULL AFTER slideVolume;

core/frontend.py

@@ -1,246 +1,745 @@
 import logging, coloredlogs
-from typing import Any, Dict, List
-from twisted.web import resource
-from twisted.web.util import redirectTo
-from twisted.web.http import Request
+from typing import Any, Dict, List, Union, Optional
+from starlette.requests import Request
+from starlette.routing import Route, Mount
+from starlette.responses import Response, PlainTextResponse, RedirectResponse
+from starlette.applications import Starlette
 from logging.handlers import TimedRotatingFileHandler
-from twisted.web.server import Session
-from zope.interface import Interface, Attribute, implementer
-from twisted.python.components import registerAdapter
 import jinja2
 import bcrypt
+import re
+import jwt
+import yaml
+from base64 import b64decode
+from enum import Enum
+from datetime import datetime, timezone
+from os import path, environ, mkdir, W_OK, access
 
 from core import CoreConfig, Utils
 from core.data import Data
 
+class PermissionOffset(Enum):
+    USER = 0 # Regular user
+    USERMOD = 1 # Can moderate other users
+    ACMOD = 2 # Can add arcades and cabs
+    SYSADMIN = 3 # Can change settings
+    # 4 - 6 reserved for future use
+    OWNER = 7 # Can do anything
 
-class IUserSession(Interface):
-    userId = Attribute("User's ID")
-    current_ip = Attribute("User's current ip address")
-    permissions = Attribute("User's permission level")
+class ShopPermissionOffset(Enum):
+    VIEW = 0 # View info and cabs
+    BOOKKEEP = 1 # View bookeeping info
+    EDITOR = 2 # Can edit name, settings
+    REGISTRAR = 3 # Can add cabs
+    # 4 - 6 reserved for future use
+    OWNER = 7 # Can do anything
 
+class ShopOwner():
+    def __init__(self, usr_id: int = 0, usr_name: str = "", perms: int = 0) -> None:
+        self.user_id = usr_id
+        self.username = usr_name
+        self.permissions = perms
 
-@implementer(IUserSession)
-class UserSession(object):
-    def __init__(self, session):
-        self.userId = 0
-        self.current_ip = "0.0.0.0"
-        self.permissions = 0
-
-
-class FrontendServlet(resource.Resource):
-    def getChild(self, name: bytes, request: Request):
-        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {name.decode()}")
-        if name == b"":
-            return self
-        return resource.Resource.getChild(self, name, request)
+class UserSession():
+    def __init__(self, usr_id: int = 0, ip: str = "", perms: int = 0, ongeki_ver: int = 7):
+        self.user_id = usr_id
+        self.current_ip = ip
+        self.permissions = perms
+        self.ongeki_version = ongeki_ver
 
+class FrontendServlet():
     def __init__(self, cfg: CoreConfig, config_dir: str) -> None:
         self.config = cfg
         log_fmt_str = "[%(asctime)s] Frontend | %(levelname)s | %(message)s"
         log_fmt = logging.Formatter(log_fmt_str)
-        self.logger = logging.getLogger("frontend")
         self.environment = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
-        self.game_list: List[Dict[str, str]] = []
-        self.children: Dict[str, Any] = {}
+        self.game_list: Dict[str, Dict[str, Any]] = {}
+        self.sn_cvt: Dict[str, str] = {}
+        
+        self.logger = logging.getLogger("frontend")
+        if not hasattr(self.logger, "inited"):
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.config.server.log_dir, "frontend"),
+                when="d",
+                backupCount=10,
+            )
+            fileHandler.setFormatter(log_fmt)
 
-        fileHandler = TimedRotatingFileHandler(
-            "{0}/{1}.log".format(self.config.server.log_dir, "frontend"),
-            when="d",
-            backupCount=10,
-        )
-        fileHandler.setFormatter(log_fmt)
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
 
-        consoleHandler = logging.StreamHandler()
-        consoleHandler.setFormatter(log_fmt)
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
 
-        self.logger.addHandler(fileHandler)
-        self.logger.addHandler(consoleHandler)
+            self.logger.setLevel(cfg.frontend.loglevel)
+            coloredlogs.install(
+                level=cfg.frontend.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
 
-        self.logger.setLevel(cfg.frontend.loglevel)
-        coloredlogs.install(
-            level=cfg.frontend.loglevel, logger=self.logger, fmt=log_fmt_str
-        )
-        registerAdapter(UserSession, Session, IUserSession)
-
-        fe_game = FE_Game(cfg, self.environment)
+            self.logger.inited = True
+        
         games = Utils.get_all_titles()
         for game_dir, game_mod in games.items():
-            if hasattr(game_mod, "frontend"):
+            if hasattr(game_mod, "frontend") and hasattr(game_mod, "index") and hasattr(game_mod, "game_codes"):
                 try:
-                    game_fe = game_mod.frontend(cfg, self.environment, config_dir)
-                    self.game_list.append({"url": game_dir, "name": game_fe.nav_name})
-                    fe_game.putChild(game_dir.encode(), game_fe)
+                    if game_mod.index.is_game_enabled(game_mod.game_codes[0], self.config, config_dir):
+                        game_fe = game_mod.frontend(cfg, self.environment, config_dir)
+                        self.game_list[game_fe.nav_name] = {"url": f"/{game_dir}", "class": game_fe }
+                        
+                        if hasattr(game_fe, "SN_PREFIX") and hasattr(game_fe, "NETID_PREFIX"):
+                            if len(game_fe.SN_PREFIX) == len(game_fe.NETID_PREFIX):
+                                for x in range(len(game_fe.SN_PREFIX)):
+                                    self.sn_cvt[game_fe.SN_PREFIX[x]] = game_fe.NETID_PREFIX[x]
 
                 except Exception as e:
                     self.logger.error(
                         f"Failed to import frontend from {game_dir} because {e}"
                     )
 
+
         self.environment.globals["game_list"] = self.game_list
-        self.putChild(b"gate", FE_Gate(cfg, self.environment))
-        self.putChild(b"user", FE_User(cfg, self.environment))
-        self.putChild(b"game", fe_game)
+        self.environment.globals["sn_cvt"] = self.sn_cvt
+        self.base = FE_Base(cfg, self.environment)
+        self.gate = FE_Gate(cfg, self.environment)
+        self.user = FE_User(cfg, self.environment)
+        self.system = FE_System(cfg, self.environment)
+        self.arcade = FE_Arcade(cfg, self.environment)
+        self.machine = FE_Machine(cfg, self.environment)
+    
+    def get_routes(self) -> List[Route]:        
+        g_routes = []
+        for nav_name, g_data in self.environment.globals["game_list"].items():
+            g_routes.append(Mount(g_data['url'], routes=g_data['class'].get_routes()))
+        return [
+            Route("/", self.base.render_GET, methods=['GET']),
+            Mount("/user", routes=[
+                Route("/", self.user.render_GET, methods=['GET']),
+                Route("/{user_id:int}", self.user.render_GET, methods=['GET']),
+                Route("/update.pw", self.user.render_POST, methods=['POST']),
+                Route("/update.name", self.user.update_username, methods=['POST']),
+                Route("/edit.card", self.user.edit_card, methods=['POST']),
+                Route("/add.card", self.user.add_card, methods=['POST']),
+                Route("/logout", self.user.render_logout, methods=['GET']),
+            ]),
+            Mount("/gate", routes=[
+                Route("/", self.gate.render_GET, methods=['GET', 'POST']),
+                Route("/gate.login", self.gate.render_login, methods=['POST']),
+                Route("/gate.create", self.gate.render_create, methods=['POST']),
+                Route("/create", self.gate.render_create_get, methods=['GET']),
+            ]),
+            Mount("/sys", routes=[
+                Route("/", self.system.render_GET, methods=['GET']),
+                Route("/lookup.user", self.system.lookup_user, methods=['GET']),
+                Route("/lookup.shop", self.system.lookup_shop, methods=['GET']),
+            ]),
+            Mount("/shop", routes=[
+                Route("/", self.arcade.render_GET, methods=['GET']),
+                Route("/{shop_id:int}", self.arcade.render_GET, methods=['GET']),
+            ]),
+            Mount("/cab", routes=[
+                Route("/", self.machine.render_GET, methods=['GET']),
+                Route("/{machine_id:int}", self.machine.render_GET, methods=['GET']),
+            ]),
+            Mount("/game", routes=g_routes),            
+            Route("/robots.txt", self.robots)
+        ]
+    
+    def startup(self) -> None:
+        self.config.update({
+            "frontend": {
+                "standalone": True, 
+                "loglevel": CoreConfig.loglevel_to_str(self.config.frontend.loglevel), 
+                "secret": self.config.frontend.secret
+            }
+        })
+        self.logger.info(f"Serving {len(self.game_list)} games")
+    
+    @classmethod    
+    async def robots(cls, request: Request) -> PlainTextResponse:
+        return PlainTextResponse("User-agent: *\nDisallow: /\n\nUser-agent: AdsBot-Google\nDisallow: /")
 
-        self.logger.info(
-            f"Ready on port {self.config.frontend.port} serving {len(fe_game.children)} games"
-        )
-
-    def render_GET(self, request):
-        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.uri.decode()}")
-        template = self.environment.get_template("core/frontend/index.jinja")
-        return template.render(
-            server_name=self.config.server.name,
-            title=self.config.server.name,
-            game_list=self.game_list,
-            sesh=vars(IUserSession(request.getSession())),
-        ).encode("utf-16")
-
-
-class FE_Base(resource.Resource):
+class FE_Base():
     """
     A Generic skeleton class that all frontend handlers should inherit from
     Initializes the environment, data, logger, config, and sets isLeaf to true
     It is expected that game implementations of this class overwrite many of these
     """
-
-    isLeaf = True
-
     def __init__(self, cfg: CoreConfig, environment: jinja2.Environment) -> None:
         self.core_config = cfg
         self.data = Data(cfg)
         self.logger = logging.getLogger("frontend")
         self.environment = environment
-        self.nav_name = "nav_name"
+        self.nav_name = "index"
+    
+    async def render_GET(self, request: Request):
+        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.url}")
+        template = self.environment.get_template("core/templates/index.jinja")
+        sesh = self.validate_session(request)
+        resp = Response(template.render(
+            server_name=self.core_config.server.name,
+            title=self.core_config.server.name,
+            game_list=self.environment.globals["game_list"],
+            sesh=vars(sesh) if sesh is not None else vars(UserSession()),
+        ), media_type="text/html; charset=utf-8")
+        
+        if sesh is None:
+            resp.delete_cookie("DIANA_SESH")
+        return resp
+    
+    def get_routes(self) -> List[Route]:
+        return []
+    
+    @classmethod
+    def test_perm(cls, permission: int, offset: Union[PermissionOffset, ShopPermissionOffset]) -> bool:
+        logging.getLogger('frontend').debug(f"{permission} vs {1 << offset.value}")
+        return permission & 1 << offset.value == 1 << offset.value
+    
+    @classmethod
+    def test_perm_minimum(cls, permission: int, offset: Union[PermissionOffset, ShopPermissionOffset]) -> bool:
+        return permission >= 1 << offset.value
+    
+    def decode_session(self, token: str) -> UserSession:
+        sesh = UserSession()
+        if not token: return sesh
+        try:
+            tk = jwt.decode(token, b64decode(self.core_config.frontend.secret), options={"verify_signature": True}, algorithms=["HS256"])
+            sesh.user_id = tk['user_id']
+            sesh.current_ip = tk['current_ip']
+            sesh.permissions = tk['permissions']
+            
+            if sesh.user_id <= 0:
+                self.logger.error("User session failed to validate due to an invalid ID!")
+                return UserSession()
+            return sesh
+        except jwt.ExpiredSignatureError:
+            self.logger.error("User session failed to validate due to an expired signature!")
+            return sesh
+        except jwt.InvalidSignatureError:
+            self.logger.error("User session failed to validate due to an invalid signature!")
+            return sesh
+        except jwt.DecodeError as e:
+            self.logger.error(f"User session failed to decode! {e}")
+            return sesh
+        except jwt.InvalidTokenError as e:
+            self.logger.error(f"User session is invalid! {e}")
+            return sesh
+        except KeyError as e:
+            self.logger.error(f"{e} missing from User session!")
+            return UserSession()
+        except Exception as e:
+            self.logger.error(f"Unknown exception occoured when decoding User session! {e}")
+            return UserSession()
+    
+    def validate_session(self, request: Request) -> Optional[UserSession]:
+        sesh = request.cookies.get('DIANA_SESH', "")
+        if not sesh:
+            return None
+        
+        usr_sesh = self.decode_session(sesh)
+        req_ip = Utils.get_ip_addr(request)
+        
+        if usr_sesh.current_ip != req_ip:
+            self.logger.error(f"User session failed to validate due to mismatched IPs! {usr_sesh.current_ip} -> {req_ip}")
+            return None
 
+        if usr_sesh.permissions <= 0 or usr_sesh.permissions > 255:
+            self.logger.error(f"User session failed to validate due to an invalid permission value! {usr_sesh.permissions}")
+            return None
+        
+        return usr_sesh
+    
+    def encode_session(self, sesh: UserSession, exp_seconds: int = 86400) -> str:
+        try:
+            return jwt.encode({ "user_id": sesh.user_id, "current_ip": sesh.current_ip, "permissions": sesh.permissions, "ongeki_version": sesh.ongeki_version, "exp": int(datetime.now(tz=timezone.utc).timestamp()) + exp_seconds }, b64decode(self.core_config.frontend.secret), algorithm="HS256")
+        except jwt.InvalidKeyError:
+            self.logger.error("Failed to encode User session because the secret is invalid!")
+            return ""
+        except Exception as e:
+            self.logger.error(f"Unknown exception occoured when encoding User session! {e}")
+            return ""
 
 class FE_Gate(FE_Base):
-    def render_GET(self, request: Request):
-        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.uri.decode()}")
-        uri: str = request.uri.decode()
+    async def render_GET(self, request: Request):
+        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.url.path}")
 
-        sesh = request.getSession()
-        usr_sesh = IUserSession(sesh)
-        if usr_sesh.userId > 0:
-            return redirectTo(b"/user", request)
-
-        if uri.startswith("/gate/create"):
-            return self.create_user(request)
-
-        if b"e" in request.args:
+        usr_sesh = self.validate_session(request)
+        if usr_sesh and usr_sesh.user_id > 0:
+            return RedirectResponse("/user/", 303)
+        
+        
+        if "e" in request.query_params:
             try:
-                err = int(request.args[b"e"][0].decode())
-            except:
+                err = int(request.query_params.get("e", ["0"])[0])
+            except Exception:
                 err = 0
 
         else:
             err = 0
 
-        template = self.environment.get_template("core/frontend/gate/gate.jinja")
-        return template.render(
+        template = self.environment.get_template("core/templates/gate/gate.jinja")
+        resp = Response(template.render(
             title=f"{self.core_config.server.name} | Login Gate",
             error=err,
-            sesh=vars(usr_sesh),
-        ).encode("utf-16")
-
-    def render_POST(self, request: Request):
-        uri = request.uri.decode()
+            sesh=vars(UserSession()),
+        ), media_type="text/html; charset=utf-8")
+        resp.delete_cookie("DIANA_SESH")
+        return resp
+    
+    async def render_login(self, request: Request):
         ip = Utils.get_ip_addr(request)
+        frm = await request.form()
+        access_code: str = frm.get("access_code", None)
+        if not access_code:
+            return RedirectResponse("/gate/?e=1", 303)
+        
+        passwd: bytes = frm.get("passwd", "").encode()
+        if passwd == b"":
+            passwd = None
 
-        if uri == "/gate/gate.login":
-            access_code: str = request.args[b"access_code"][0].decode()
-            passwd: bytes = request.args[b"passwd"][0]
-            if passwd == b"":
-                passwd = None
+        uid = await self.data.card.get_user_id_from_card(access_code)
+        if uid is None:
+            self.logger.debug(f"Failed to find user for card {access_code}")
+            return RedirectResponse("/gate/?e=1", 303)
 
-            uid = self.data.card.get_user_id_from_card(access_code)
-            if uid is None:
-                return redirectTo(b"/gate?e=1", request)
+        user = await self.data.user.get_user(uid)
+        if user is None:
+            self.logger.error(f"Failed to load user {uid}")
+            return RedirectResponse("/gate/?e=1", 303)
 
-            if passwd is None:
-                sesh = self.data.user.check_password(uid)
+        if passwd is None:
+            sesh = await self.data.user.check_password(uid)
 
-                if sesh is not None:
-                    return redirectTo(
-                        f"/gate/create?ac={access_code}".encode(), request
-                    )
-                return redirectTo(b"/gate?e=1", request)
+            if sesh is not None:
+                return RedirectResponse(f"/gate/create?ac={access_code}", 303)
+            
+            return RedirectResponse("/gate/?e=1", 303)
 
-            if not self.data.user.check_password(uid, passwd):
-                return redirectTo(b"/gate?e=1", request)
+        if not await self.data.user.check_password(uid, passwd):
+            self.logger.debug(f"Failed password for access code {access_code}")
+            return RedirectResponse("/gate/?e=1", 303)
 
-            self.logger.info(f"Successful login of user {uid} at {ip}")
+        self.logger.info(f"Successful login of user {uid} at {ip}")
 
-            sesh = request.getSession()
-            usr_sesh = IUserSession(sesh)
-            usr_sesh.userId = uid
-            usr_sesh.current_ip = ip
+        sesh = UserSession()
+        sesh.user_id = uid
+        sesh.current_ip = ip
+        sesh.permissions = user['permissions']
+        
+        usr_sesh = self.encode_session(sesh)
+        self.logger.debug(f"Created session with JWT {usr_sesh}")
+        resp = RedirectResponse("/user/", 303)
+        resp.set_cookie("DIANA_SESH", usr_sesh)
 
-            return redirectTo(b"/user", request)
+        return resp
 
-        elif uri == "/gate/gate.create":
-            access_code: str = request.args[b"access_code"][0].decode()
-            username: str = request.args[b"username"][0]
-            email: str = request.args[b"email"][0].decode()
-            passwd: str = request.args[b"passwd"][0]
+    async def render_create(self, request: Request):
+        ip = Utils.get_ip_addr(request)
+        frm = await request.form()
+        access_code: str = frm.get("access_code", "")
+        username: str = frm.get("username", "")
+        email: str = frm.get("email", "")
+        passwd: bytes = frm.get("passwd", "").encode()
 
-            uid = self.data.card.get_user_id_from_card(access_code)
-            if uid is None:
-                return redirectTo(b"/gate?e=1", request)
+        if not access_code or not username or not email or not passwd:
+            return RedirectResponse("/gate/?e=1", 303)
 
-            salt = bcrypt.gensalt()
-            hashed = bcrypt.hashpw(passwd, salt)
+        uid = await self.data.card.get_user_id_from_card(access_code)
+        if uid is None:
+            return RedirectResponse("/gate/?e=1", 303)
 
-            result = self.data.user.create_user(
-                uid, username, email, hashed.decode(), 1
-            )
-            if result is None:
-                return redirectTo(b"/gate?e=3", request)
+        salt = bcrypt.gensalt()
+        hashed = bcrypt.hashpw(passwd, salt)
 
-            if not self.data.user.check_password(uid, passwd.encode()):
-                return redirectTo(b"/gate", request)
+        result = await self.data.user.create_user(
+            uid, username, email.lower(), hashed.decode(), 1
+        )
+        if result is None:
+            return RedirectResponse("/gate/?e=3", 303)
 
-            return redirectTo(b"/user", request)
+        if not await self.data.user.check_password(uid, passwd):
+            return RedirectResponse("/gate/", 303)
+        
+        sesh = UserSession()
+        sesh.user_id = uid
+        sesh.current_ip = ip
+        sesh.permissions = 1
+        
+        usr_sesh = self.encode_session(sesh)
+        self.logger.debug(f"Created session with JWT {usr_sesh}")
+        resp = RedirectResponse("/user/", 303)
+        resp.set_cookie("DIANA_SESH", usr_sesh)
 
-        else:
-            return b""
+        return resp
 
-    def create_user(self, request: Request):
-        if b"ac" not in request.args or len(request.args[b"ac"][0].decode()) != 20:
-            return redirectTo(b"/gate?e=2", request)
+    async def render_create_get(self, request: Request):
+        ac = request.query_params.get("ac", "")
+        if len(ac) != 20:
+            return RedirectResponse("/gate/?e=2", 303)
 
-        ac = request.args[b"ac"][0].decode()
+        card = await self.data.card.get_card_by_access_code(ac)
+        if card is None:
+            return RedirectResponse("/gate/?e=1", 303)
+        
+        user = await self.data.user.get_user(card['user'])
+        if user is None:
+            self.logger.warning(f"Card {ac} exists with no/invalid associated user ID {card['user']}")
+            return RedirectResponse("/gate/?e=0", 303)
 
-        template = self.environment.get_template("core/frontend/gate/create.jinja")
-        return template.render(
+        if user['password'] is not None:
+            return RedirectResponse("/gate/?e=1", 303)
+
+        template = self.environment.get_template("core/templates/gate/create.jinja")
+        return Response(template.render(
             title=f"{self.core_config.server.name} | Create User",
             code=ac,
-            sesh={"userId": 0},
-        ).encode("utf-16")
-
+            sesh={"user_id": 0, "permissions": 0},
+        ), media_type="text/html; charset=utf-8")
 
 class FE_User(FE_Base):
-    def render_GET(self, request: Request):
-        template = self.environment.get_template("core/frontend/user/index.jinja")
+    async def render_GET(self, request: Request):
+        uri = request.url.path
+        user_id = request.path_params.get('user_id', None)
+        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {uri}")
+        template = self.environment.get_template("core/templates/user/index.jinja")
 
-        sesh: Session = request.getSession()
-        usr_sesh = IUserSession(sesh)
-        if usr_sesh.userId == 0:
-            return redirectTo(b"/gate", request)
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh:
+            return RedirectResponse("/gate/", 303)
+        
+        if user_id:
+            if not self.test_perm(usr_sesh.permissions, PermissionOffset.USERMOD) and user_id != usr_sesh.user_id:
+                self.logger.warn(f"User {usr_sesh.user_id} does not have permission to view user {user_id}")
+                return RedirectResponse("/user/", 303)
+        
+        else:
+            user_id = usr_sesh.user_id
+        
+        user = await self.data.user.get_user(user_id)
+        if user is None:
+            self.logger.debug(f"User {user_id} not found")
+            return RedirectResponse("/user/", 303)
+        
+        cards = await self.data.card.get_user_cards(user_id)
+        
+        card_data = []
+        arcade_data = []
 
-        return template.render(
-            title=f"{self.core_config.server.name} | Account", sesh=vars(usr_sesh)
-        ).encode("utf-16")
+        for c in cards:
+            if c['is_locked']:
+                status = 'Locked'
+            elif c['is_banned']:
+                status = 'Banned'
+            else:
+                status = 'Active'
+            
+            #idm = c['idm']
+            ac = c['access_code']
 
+            if ac.startswith("5"): #or idm is not None:
+                c_type = "AmusementIC"
+            elif ac.startswith("3"):
+                c_type = "Banapass"
+            elif ac.startswith("010"):                
+                c_type = "Aime" # TODO: Aime verification
+            elif ac.startswith("0008"):
+                c_type = "Generated AIC"
+            else:
+                c_type = "Unknown"
+            
+            card_data.append({
+                'access_code': ac, 
+                'status': status, 
+                'chip_id': "", #None if c['chip_id'] is None else f"{c['chip_id']:X}", 
+                'idm': "", 
+                'type': c_type, 
+                "memo": ""
+            })
 
-class FE_Game(FE_Base):
-    isLeaf = False
-    children: Dict[str, Any] = {}
+        if "e" in request.query_params:
+            try:
+                err = int(request.query_params.get("e", 0))
+            except Exception:
+                err = 0
 
-    def getChild(self, name: bytes, request: Request):
-        if name == b"":
-            return self
-        return resource.Resource.getChild(self, name, request)
+        else:
+            err = 0
+        
+        if "s" in request.query_params:
+            try:
+                succ = int(request.query_params.get("s", 0))
+            except Exception:
+                succ = 0
 
-    def render_GET(self, request: Request) -> bytes:
-        return redirectTo(b"/user", request)
+        else:
+            succ = 0
+
+        return Response(template.render(
+            title=f"{self.core_config.server.name} | Account", 
+            sesh=vars(usr_sesh), 
+            cards=card_data,
+            error=err,
+            success=succ,
+            username=user['username'],
+            arcades=arcade_data
+        ), media_type="text/html; charset=utf-8")
+    
+    async def render_logout(self, request: Request):
+        resp = RedirectResponse("/gate/", 303)
+        resp.delete_cookie("DIANA_SESH")
+        return resp
+    
+    async def edit_card(self, request: Request) -> RedirectResponse:
+        return RedirectResponse("/user/", 303)
+    
+    async def add_card(self, request: Request) -> RedirectResponse:
+        return RedirectResponse("/user/", 303)
+
+    async def render_POST(self, request: Request):
+        frm = await request.form()
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.USERMOD):
+            return RedirectResponse("/gate/", 303)
+        
+        old_pw: str = frm.get('current_pw', None)
+        pw1: str = frm.get('password1', None)
+        pw2: str = frm.get('password2', None)
+
+        if old_pw is None or pw1 is None or pw2 is None:
+            return RedirectResponse("/user/?e=4", 303)
+        
+        if pw1 != pw2:
+            return RedirectResponse("/user/?e=6", 303)
+        
+        if not await self.data.user.check_password(usr_sesh.user_id, old_pw.encode()):
+            return RedirectResponse("/user/?e=5", 303)
+        
+        if len(pw1) < 10 or not any(ele.isupper() for ele in pw1) or not any(ele.islower() for ele in pw1) \
+            or not any(ele.isdigit() for ele in pw1) or not any(not ele.isalnum() for ele in pw1):
+            return RedirectResponse("/user/?e=7", 303)
+        
+        salt = bcrypt.gensalt()
+        hashed = bcrypt.hashpw(pw1.encode(), salt)
+        if not await self.data.user.change_password(usr_sesh.user_id, hashed.decode()):
+            return RedirectResponse("/gate/?e=1", 303)
+        
+        return RedirectResponse("/user/?s=1", 303)
+        
+    async def update_username(self, request: Request):
+        frm = await request.form()
+        new_name: bytes = frm.get('new_name', "")
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.USERMOD):
+            return RedirectResponse("/gate/", 303)
+        
+        if new_name is None or not new_name:
+            return RedirectResponse("/user/?e=4", 303)
+        
+        if len(new_name) > 10:
+            return RedirectResponse("/user/?e=8", 303)
+
+        if not await self.data.user.change_username(usr_sesh.user_id, new_name):
+            return RedirectResponse("/user/?e=8", 303)
+        
+        return RedirectResponse("/user/?s=2", 303)
+
+class FE_System(FE_Base):
+    async def render_GET(self, request: Request):
+        template = self.environment.get_template("core/templates/sys/index.jinja")
+        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.url.path}")
+
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh or not self.test_perm_minimum(usr_sesh.permissions, PermissionOffset.USERMOD):
+            return RedirectResponse("/gate/", 303)
+        
+        return Response(template.render(
+            title=f"{self.core_config.server.name} | System", 
+            sesh=vars(usr_sesh), 
+            usrlist=[],
+        ), media_type="text/html; charset=utf-8")
+        
+    async def lookup_user(self, request: Request):
+        template = self.environment.get_template("core/templates/sys/index.jinja")
+        usrlist: List[Dict] = []
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.USERMOD):
+            return RedirectResponse("/gate/", 303)
+        
+        uid_search = request.query_params.get("usrId",  None)
+        email_search = request.query_params.get("usrEmail", None)
+        uname_search = request.query_params.get("usrName", None)
+
+        if uid_search:
+            u = await self.data.user.get_user(uid_search)
+            if u is not None:
+                usrlist.append(u._asdict())
+
+        elif email_search:
+            u = await self.data.user.find_user_by_email(email_search)
+            if u is not None:
+                usrlist.append(u._asdict())
+
+        elif uname_search:
+            ul = await self.data.user.find_user_by_username(uname_search)
+            for u in ul:
+                usrlist.append(u._asdict())
+
+        return Response(template.render(
+            title=f"{self.core_config.server.name} | System", 
+            sesh=vars(usr_sesh), 
+            usrlist=usrlist,
+            shoplist=[],
+        ), media_type="text/html; charset=utf-8")
+
+    async def lookup_shop(self, request: Request):
+        shoplist = []
+        template = self.environment.get_template("core/templates/sys/index.jinja")
+        
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.ACMOD):
+            return RedirectResponse("/gate/", 303)
+        
+        shopid_search = request.query_params.get("shopId",  None)
+        sn_search = request.query_params.get("serialNum", None)
+        
+        if shopid_search:
+            if shopid_search.isdigit():
+                shopid_search = int(shopid_search)
+                try:
+                    sinfo = await self.data.arcade.get_arcade(shopid_search)
+                except Exception as e:
+                    self.logger.error(f"Failed to fetch shop info for shop {shopid_search} in lookup_shop - {e}")
+                    sinfo = None
+                if sinfo:
+                    shoplist.append({
+                        "name": sinfo['name'],
+                        "id": sinfo['id']
+                    })
+            
+            else:
+                return Response(template.render(
+                    title=f"{self.core_config.server.name} | System", 
+                    sesh=vars(usr_sesh),
+                    usrlist=[],
+                    shoplist=shoplist,
+                    error=4
+                ), media_type="text/html; charset=utf-8")
+        
+        if sn_search:
+            sn_search = sn_search.upper().replace("-", "").strip()
+            if sn_search.isdigit() and len(sn_search) == 12:
+                prefix = sn_search[:4]
+                suffix = sn_search[5:]
+                
+                netid_prefix = self.environment.globals["sn_cvt"].get(prefix, "")                
+                sn_search = netid_prefix + suffix
+            
+            if re.match(r"^AB[DGL]N\d{7}$", sn_search) or re.match(r"^A\d{2}[EX]\d{2}[A-Z]\d{4,8}$", sn_search):
+                cabinfo = await self.data.arcade.get_machine(sn_search)
+                if cabinfo is None: sinfo = None
+                else:
+                    sinfo = await self.data.arcade.get_arcade(cabinfo['arcade'])
+                if sinfo:
+                    shoplist.append({
+                        "name": sinfo['name'],
+                        "id": sinfo['id']
+                    })
+            
+            else:
+                return Response(template.render(
+                    title=f"{self.core_config.server.name} | System", 
+                    sesh=vars(usr_sesh),
+                    usrlist=[],
+                    shoplist=shoplist,
+                    error=10
+                ), media_type="text/html; charset=utf-8")
+            
+        
+        return Response(template.render(
+            title=f"{self.core_config.server.name} | System", 
+            sesh=vars(usr_sesh), 
+            usrlist=[],
+            shoplist=shoplist,
+        ), media_type="text/html; charset=utf-8")
+
+class FE_Arcade(FE_Base):
+    async def render_GET(self, request: Request):
+        template = self.environment.get_template("core/templates/arcade/index.jinja")
+        shop_id = request.path_params.get('shop_id', None)
+        
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.ACMOD):
+            self.logger.warn(f"User {usr_sesh.user_id} does not have permission to view shops!")
+            return RedirectResponse("/gate/", 303)
+        
+        if not shop_id:
+            return Response(template.render(
+                title=f"{self.core_config.server.name} | Arcade", 
+                sesh=vars(usr_sesh),
+            ), media_type="text/html; charset=utf-8")
+        
+        sinfo = await self.data.arcade.get_arcade(shop_id)
+        if not sinfo:
+            return Response(template.render(
+                title=f"{self.core_config.server.name} | Arcade", 
+                sesh=vars(usr_sesh),
+            ), media_type="text/html; charset=utf-8")
+        
+        cabs = await self.data.arcade.get_arcade_machines(shop_id)
+        cablst = []
+        if cabs:
+            for x in cabs:
+                cablst.append({
+                    "id": x['id'],
+                    "serial": x['serial'],
+                    "game": x['game'],
+                })
+        
+        return Response(template.render(
+            title=f"{self.core_config.server.name} | Arcade", 
+            sesh=vars(usr_sesh),
+            arcade={
+                "name": sinfo['name'],
+                "id": sinfo['id'],
+                "cabs": cablst
+            }
+            
+        ), media_type="text/html; charset=utf-8")
+
+class FE_Machine(FE_Base):
+    async def render_GET(self, request: Request):
+        template = self.environment.get_template("core/templates/machine/index.jinja")
+        cab_id = request.path_params.get('cab_id', None)
+        
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.ACMOD):
+            self.logger.warn(f"User {usr_sesh.user_id} does not have permission to view shops!")
+            return RedirectResponse("/gate/", 303)
+        
+        if not cab_id:
+            return Response(template.render(
+                title=f"{self.core_config.server.name} | Machine", 
+                sesh=vars(usr_sesh),
+            ), media_type="text/html; charset=utf-8")
+        
+        return Response(template.render(
+            title=f"{self.core_config.server.name} | Machine", 
+            sesh=vars(usr_sesh),
+            arcade={}
+        ), media_type="text/html; charset=utf-8")
+
+cfg_dir = environ.get("DIANA_CFG_DIR", "config")
+cfg: CoreConfig = CoreConfig()
+if path.exists(f"{cfg_dir}/core.yaml"):
+    cfg.update(yaml.safe_load(open(f"{cfg_dir}/core.yaml")))
+
+if not path.exists(cfg.server.log_dir):
+    mkdir(cfg.server.log_dir)
+
+if not access(cfg.server.log_dir, W_OK):
+    print(
+        f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
+    )
+    exit(1)
+
+fe = FrontendServlet(cfg, cfg_dir)
+app = Starlette(cfg.server.is_develop, fe.get_routes(), on_startup=[fe.startup])