Merge branch 'develop'

core/__init__.py

@@ -4,3 +4,4 @@ from core.aimedb import AimedbFactory
 from core.title import TitleServlet
 from core.utils import Utils
 from core.mucha import MuchaServlet
+from core.frontend import FrontendServlet

core/allnet.py

@@ -95,14 +95,19 @@ class AllnetServlet:
     def handle_poweron(self, request: Request, _: Dict):
         request_ip = request.getClientAddress().host
         try:
-            req = AllnetPowerOnRequest(self.allnet_req_to_dict(request.content.getvalue())[0])
+            req_dict = self.allnet_req_to_dict(request.content.getvalue())
+            if req_dict is None:
+                raise AllnetRequestException()
+
+            req = AllnetPowerOnRequest(req_dict[0])
             # Validate the request. Currently we only validate the fields we plan on using
 
             if not req.game_id or not req.ver or not req.token or not req.serial or not req.ip:
                 raise AllnetRequestException(f"Bad auth request params from {request_ip} - {vars(req)}")
         
         except AllnetRequestException as e:
-            self.logger.error(e)
+            if e.message != "":
+                self.logger.error(e)
             return b""
         
         if req.format_ver == 3:
@@ -155,14 +160,19 @@ class AllnetServlet:
     def handle_dlorder(self, request: Request, _: Dict):
         request_ip = request.getClientAddress().host
         try:
-            req = AllnetDownloadOrderRequest(self.allnet_req_to_dict(request.content.getvalue())[0])
+            req_dict = self.allnet_req_to_dict(request.content.getvalue())
+            if req_dict is None:
+                raise AllnetRequestException()
+
+            req = AllnetDownloadOrderRequest(req_dict[0])
             # Validate the request. Currently we only validate the fields we plan on using
 
             if not req.game_id or not req.ver or not req.serial:
                 raise AllnetRequestException(f"Bad download request params from {request_ip} - {vars(req)}")
         
         except AllnetRequestException as e:
-            self.logger.error(e)
+            if e.message != "":
+                self.logger.error(e)
             return b""
 
         resp = AllnetDownloadOrderResponse()
@@ -234,6 +244,10 @@ class AllnetServlet:
         self.logger.debug(f"response {vars(resp)}")
         return resp_str.encode("utf-8")
 
+    def handle_naomitest(self, request: Request, _: Dict) -> bytes:
+        self.logger.info(f"Ping from {request.getClientAddress().host}")
+        return b"naomi ok"
+
     def kvp_to_dict(self, kvp: List[str]) -> List[Dict[str, Any]]:
         ret: List[Dict[str, Any]] = []
         for x in kvp:
@@ -261,7 +275,7 @@ class AllnetServlet:
             return self.kvp_to_dict(sections)
 
         except Exception as e:
-            self.logger.error(e)
+            self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
             return None
 
     def allnet_req_to_dict(self, data: str) -> Optional[List[Dict[str, Any]]]:
@@ -276,7 +290,7 @@ class AllnetServlet:
             return self.kvp_to_dict(sections)
 
         except Exception as e:
-            self.logger.error(e)
+            self.logger.error(f"allnet_req_to_dict: {e} while parsing {data}")
             return None
 
     def dict_to_http_form_string(self, data:List[Dict[str, Any]], crlf: bool = False, trailing_newline: bool = True) -> Optional[str]:
@@ -303,7 +317,7 @@ class AllnetServlet:
             return urlencode
             
         except Exception as e:
-            self.logger.error(e)
+            self.logger.error(f"dict_to_http_form_string: {e} while parsing {data}")
             return None
 
 class AllnetPowerOnRequest():
@@ -403,6 +417,6 @@ class BillingResponse():
         # YYYY -> 4 digit year, MM -> 2 digit month, C -> Playcount during that period
 
 class AllnetRequestException(Exception):
-    def __init__(self, message="Allnet Request Error") -> None:
+    def __init__(self, message="") -> None:
         self.message = message
         super().__init__(self.message)

core/config.py

@@ -194,6 +194,7 @@ class CoreConfig(dict):
         self.allnet = AllnetConfig(self)
         self.billing = BillingConfig(self)
         self.aimedb = AimedbConfig(self)
+        self.mucha = MuchaConfig(self)
 
     @classmethod
     def str_to_loglevel(cls, level_str: str):

core/const.py

@@ -31,6 +31,61 @@ class KeychipPlatformsCodes():
     NU = ("A60E", "A60E", "A60E")
     NUSX = ("A61X", "A69X")
     ALLS = "A63E"
-    
-class RegionIDs(Enum):
-    pass
+
+class AllnetCountryCode(Enum):
+    JAPAN = "JPN"
+    UNITED_STATES = "USA"
+    HONG_KONG = "HKG"
+    SINGAPORE = "SGP"
+    SOUTH_KOREA = "KOR"
+    CHINA = "CHN"
+
+class AllnetJapanRegionId(Enum):
+    NONE = 0
+    AICHI = 1
+    AOMORI = 2
+    AKITA = 3
+    ISHIKAWA = 4
+    IBARAKI = 5
+    IWATE = 6
+    EHIME = 7
+    OITA = 8
+    OSAKA = 9
+    OKAYAMA = 10
+    OKINAWA = 11
+    KAGAWA = 12
+    KAGOSHIMA = 13
+    KANAGAWA = 14
+    GIFU = 15
+    KYOTO = 16
+    KUMAMOTO = 17
+    GUNMA = 18
+    KOCHI = 19
+    SAITAMA = 20
+    SAGA = 21
+    SHIGA = 22
+    SHIZUOKA = 23
+    SHIMANE = 24
+    CHIBA = 25
+    TOKYO = 26
+    TOKUSHIMA = 27
+    TOCHIGI = 28
+    TOTTORI = 29
+    TOYAMA = 30
+    NAGASAKI = 31
+    NAGANO = 32
+    NARA = 33
+    NIIGATA = 34
+    HYOGO = 35
+    HIROSHIMA = 36
+    FUKUI = 37
+    FUKUOKA = 38
+    FUKUSHIMA = 39
+    HOKKAIDO = 40
+    MIE = 41
+    MIYAGI = 42
+    MIYAZAKI = 43
+    YAMAGATA = 44
+    YAMAGUCHI = 45
+    YAMANASHI = 46
+    WAKAYAMA = 47

core/data/database.py

@@ -31,7 +31,7 @@ class Data:
         self.arcade = ArcadeData(self.config, self.session)
         self.card = CardData(self.config, self.session)
         self.base = BaseData(self.config, self.session)
-        self.schema_ver_latest = 1
+        self.schema_ver_latest = 2
 
         log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
         log_fmt = logging.Formatter(log_fmt_str)

core/data/schema/user.py

@@ -1,9 +1,14 @@
 from enum import Enum
 from typing import Dict, Optional
-from sqlalchemy import Table, Column
+from sqlalchemy import Table, Column, and_
 from sqlalchemy.types import Integer, String, TIMESTAMP
 from sqlalchemy.sql.schema import ForeignKey
 from sqlalchemy.sql import func
+from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.sql import func, select, Delete
+from uuid import uuid4
+from datetime import datetime, timedelta
+from sqlalchemy.engine import Row
 
 from core.data.schema.base import BaseData, metadata
 
@@ -26,6 +31,7 @@ frontend_session = Table(
     metadata,
     Column("id", Integer, primary_key=True, unique=True),
     Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("ip", String(15)),
     Column('session_cookie', String(32), nullable=False, unique=True),
     Column("expires", TIMESTAMP, nullable=False),
     mysql_charset='utf8mb4'
@@ -37,21 +43,83 @@ class PermissionBits(Enum):
     PermSysAdmin = 4
 
 class UserData(BaseData):
-    def create_user(self, username: str = None, email: str = None, password: str = None) -> Optional[int]:
-
+    def create_user(self, id: int = None, username: str = None, email: str = None, password: str = None, permission: int = 1) -> Optional[int]:
         if email is None:
-            permission = None
-        else:
-            permission = 0
+            permission = 1
 
-        sql = aime_user.insert().values(username=username, email=email, password=password, permissions=permission)
+        if id is None:
+            sql = insert(aime_user).values(
+                username=username, 
+                email=email, 
+                password=password, 
+                permissions=permission
+            )
+        else:
+            sql = insert(aime_user).values(
+                id=id, 
+                username=username, 
+                email=email, 
+                password=password, 
+                permissions=permission
+            )
+
+        conflict = sql.on_duplicate_key_update(
+            username=username, 
+            email=email, 
+            password=password, 
+            permissions=permission
+        )
         
-        result = self.execute(sql)
+        result = self.execute(conflict)
         if result is None: return None
         return result.lastrowid
     
+    def login(self, user_id: int, passwd: bytes = None, ip: str = "0.0.0.0") -> Optional[str]:
+        sql = select(aime_user).where(and_(aime_user.c.id == user_id, aime_user.c.password == passwd))
+
+        result = self.execute(sql)
+        if result is None: return None
+        
+        usr = result.fetchone()
+        if usr is None: return None
+
+        return self.create_session(user_id, ip)
+    
+    def check_session(self, cookie: str, ip: str = "0.0.0.0") -> Optional[Row]:
+        sql = select(frontend_session).where(
+            and_(
+                frontend_session.c.session_cookie == cookie,
+                frontend_session.c.ip == ip
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None: return None        
+        return result.fetchone()
+    
+    def delete_session(self, session_id: int) -> bool:
+        sql = Delete(frontend_session).where(frontend_session.c.id == session_id)
+
+        result = self.execute(sql)
+        if result is None: return False
+        return True
+
+    def create_session(self, user_id: int, ip: str = "0.0.0.0", expires: datetime = datetime.now() + timedelta(days=1)) -> Optional[str]:
+        cookie = uuid4().hex
+
+        sql = insert(frontend_session).values(
+            user = user_id,
+            ip = ip,
+            session_cookie = cookie,
+            expires = expires
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return cookie
+
     def reset_autoincrement(self, ai_value: int) -> None:
-        # Didn't feel like learning how to do this the right way
-        # if somebody wants a free PR go nuts I guess
+        # ALTER TABLE isn't in sqlalchemy so we do this the ugly way
         sql = f"ALTER TABLE aime_user AUTO_INCREMENT={ai_value}"
         self.execute(sql)

core/data/schema/versions/CORE_1_rollback.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `frontend_session`
+DROP COLUMN `ip`;

core/data/schema/versions/CORE_2_upgrade.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `frontend_session`
+ADD `ip` CHAR(15);

core/data/schema/versions/SBZV_1_rollback.sql

@@ -0,0 +1,9 @@
+SET FOREIGN_KEY_CHECKS=0;
+ALTER TABLE diva_score DROP COLUMN edition;
+ALTER TABLE diva_playlog DROP COLUMN edition;
+
+ALTER TABLE diva_score DROP FOREIGN KEY diva_score_ibfk_1;
+ALTER TABLE diva_score DROP CONSTRAINT diva_score_uk;
+ALTER TABLE diva_score ADD CONSTRAINT diva_score_uk UNIQUE (user, pv_id, difficulty);
+ALTER TABLE diva_score ADD CONSTRAINT diva_score_ibfk_1 FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SBZV_2_rollback.sql

@@ -0,0 +1,17 @@
+ALTER TABLE diva_profile_shop DROP COLUMN c_itm_eqp_ary;
+ALTER TABLE diva_profile_shop DROP COLUMN ms_itm_flg_ary;
+
+ALTER TABLE diva_profile DROP COLUMN use_pv_mdl_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_mdl_pri;
+ALTER TABLE diva_profile DROP COLUMN use_pv_skn_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_btn_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_sld_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_chn_sld_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_sldr_tch_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_mdl_eqp VARCHAR(8) NOT NULL DEFAULT "true" AFTER sort_kind;
+ALTER TABLE diva_profile ADD COLUMN use_pv_btn_se_eqp VARCHAR(8) NOT NULL DEFAULT "true" AFTER use_pv_mdl_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_sld_se_eqp VARCHAR(8) NOT NULL DEFAULT "false" AFTER use_pv_btn_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_chn_sld_se_eqp VARCHAR(8) NOT NULL DEFAULT "false" AFTER use_pv_sld_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_sldr_tch_se_eqp VARCHAR(8) NOT NULL DEFAULT "false" AFTER use_pv_chn_sld_se_eqp;
+
+DROP TABLE IF EXISTS `diva_profile_pv_customize`;

core/data/schema/versions/SBZV_2_upgrade.sql

@@ -0,0 +1,9 @@
+SET FOREIGN_KEY_CHECKS=0;
+ALTER TABLE diva_score ADD COLUMN edition int(11) DEFAULT 0 AFTER difficulty;
+ALTER TABLE diva_playlog ADD COLUMN edition int(11) DEFAULT 0 AFTER difficulty;
+
+ALTER TABLE diva_score DROP FOREIGN KEY diva_score_ibfk_1;
+ALTER TABLE diva_score DROP CONSTRAINT diva_score_uk;
+ALTER TABLE diva_score ADD CONSTRAINT diva_score_uk UNIQUE (user, pv_id, difficulty, edition);
+ALTER TABLE diva_score ADD CONSTRAINT diva_score_ibfk_1 FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SBZV_3_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE diva_profile DROP COLUMN passwd_stat;
+ALTER TABLE diva_profile DROP COLUMN passwd;
+ALTER TABLE diva_profile MODIFY player_name VARCHAR(8);

core/data/schema/versions/SBZV_3_upgrade.sql

@@ -0,0 +1,33 @@
+ALTER TABLE diva_profile_shop ADD COLUMN c_itm_eqp_ary varchar(59) DEFAULT "-999,-999,-999,-999,-999,-999,-999,-999,-999,-999,-999,-999";
+ALTER TABLE diva_profile_shop ADD COLUMN ms_itm_flg_ary varchar(59) DEFAULT "-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1";
+
+ALTER TABLE diva_profile DROP COLUMN use_pv_mdl_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_btn_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_sld_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_chn_sld_se_eqp;
+ALTER TABLE diva_profile DROP COLUMN use_pv_sldr_tch_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_mdl_eqp BOOLEAN NOT NULL DEFAULT true AFTER sort_kind;
+ALTER TABLE diva_profile ADD COLUMN use_mdl_pri BOOLEAN NOT NULL DEFAULT false AFTER use_pv_mdl_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_skn_eqp BOOLEAN NOT NULL DEFAULT false AFTER use_mdl_pri;
+ALTER TABLE diva_profile ADD COLUMN use_pv_btn_se_eqp BOOLEAN NOT NULL DEFAULT true AFTER use_pv_skn_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_sld_se_eqp BOOLEAN NOT NULL DEFAULT false AFTER use_pv_btn_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_chn_sld_se_eqp BOOLEAN NOT NULL DEFAULT false AFTER use_pv_sld_se_eqp;
+ALTER TABLE diva_profile ADD COLUMN use_pv_sldr_tch_se_eqp BOOLEAN NOT NULL DEFAULT false AFTER use_pv_chn_sld_se_eqp;
+
+
+CREATE TABLE diva_profile_pv_customize (
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    user INT NOT NULL,
+    version INT NOT NULL,
+    pv_id INT NOT NULL,
+    mdl_eqp_ary VARCHAR(14) DEFAULT '-999,-999,-999',
+    c_itm_eqp_ary VARCHAR(59) DEFAULT '-999,-999,-999,-999,-999,-999,-999,-999,-999,-999,-999,-999',
+    ms_itm_flg_ary VARCHAR(59) DEFAULT '-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1',
+    skin INT DEFAULT '-1',
+    btn_se INT DEFAULT '-1',
+    sld_se INT DEFAULT '-1',
+    chsld_se INT DEFAULT '-1',
+    sldtch_se INT DEFAULT '-1',
+    UNIQUE KEY diva_profile_pv_customize_uk (user, version, pv_id),
+    CONSTRAINT diva_profile_pv_customize_ibfk_1 FOREIGN KEY (user) REFERENCES aime_user (id) ON DELETE CASCADE ON UPDATE CASCADE
+);

core/data/schema/versions/SBZV_4_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE diva_profile ADD COLUMN passwd_stat INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE diva_profile ADD COLUMN passwd VARCHAR(12) NOT NULL DEFAULT "**********";
+ALTER TABLE diva_profile MODIFY player_name VARCHAR(10);

core/data/schema/versions/SDBT_1_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE chuni_static_music CHANGE COLUMN worldsEndTag worldsEndTag VARCHAR(20) NULL DEFAULT NULL ;

core/data/schema/versions/SDBT_2_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE chuni_score_course DROP COLUMN theoryCount, DROP COLUMN orderId, DROP COLUMN playerRating;

core/data/schema/versions/SDBT_2_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE chuni_static_music CHANGE COLUMN worldsEndTag worldsEndTag VARCHAR(7) NULL DEFAULT NULL ;

core/data/schema/versions/SDBT_3_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE chuni_score_course ADD theoryCount int(11), ADD orderId int(11), ADD playerRating int(11);

core/data/schema/versions/SDEZ_1_rollback.sql

@@ -0,0 +1,3 @@
+SET FOREIGN_KEY_CHECKS=0;
+ALTER TABLE mai2_playlog DROP COLUMN trialPlayAchievement;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDEZ_2_upgrade.sql

@@ -0,0 +1,3 @@
+SET FOREIGN_KEY_CHECKS=0;
+ALTER TABLE mai2_playlog ADD trialPlayAchievement INT NULL;
+SET FOREIGN_KEY_CHECKS=1;

core/frontend.py

@@ -0,0 +1,181 @@
+import logging, coloredlogs
+from typing import Any, Dict, List
+from twisted.web import resource
+from twisted.web.util import redirectTo
+from twisted.web.http import Request
+from logging.handlers import TimedRotatingFileHandler
+import jinja2
+import bcrypt
+
+from core.config import CoreConfig
+from core.data import Data
+from core.utils import Utils
+
+class FrontendServlet(resource.Resource):
+    def getChild(self, name: bytes, request: Request):
+        self.logger.debug(f"{request.getClientIP()} -> {name.decode()}")
+        if name == b'':
+            return self
+        return resource.Resource.getChild(self, name, request)
+
+    def __init__(self, cfg: CoreConfig, config_dir: str) -> None:
+        self.config = cfg
+        log_fmt_str = "[%(asctime)s] Frontend | %(levelname)s | %(message)s"
+        log_fmt = logging.Formatter(log_fmt_str)
+        self.logger = logging.getLogger("frontend")
+        self.environment = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
+        self.game_list: List[Dict[str, str]] = []
+        self.children: Dict[str, Any] = {}
+
+        fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.log_dir, "frontend"), when="d", backupCount=10)
+        fileHandler.setFormatter(log_fmt)
+        
+        consoleHandler = logging.StreamHandler()
+        consoleHandler.setFormatter(log_fmt)
+
+        self.logger.addHandler(fileHandler)
+        self.logger.addHandler(consoleHandler)
+        
+        self.logger.setLevel(cfg.frontend.loglevel)
+        coloredlogs.install(level=cfg.frontend.loglevel, logger=self.logger, fmt=log_fmt_str)
+
+        fe_game = FE_Game(cfg, self.environment)
+        games = Utils.get_all_titles()
+        for game_dir, game_mod in games.items():
+            if hasattr(game_mod, "frontend"):
+                try:
+                    game_fe = game_mod.frontend(cfg, self.environment, config_dir)
+                    self.game_list.append({"url": game_dir, "name": game_fe.nav_name})
+                    fe_game.putChild(game_dir.encode(), game_fe)
+                except:
+                    raise
+        
+        self.environment.globals["game_list"] = self.game_list
+        self.putChild(b"gate", FE_Gate(cfg, self.environment))
+        self.putChild(b"user", FE_User(cfg, self.environment))
+        self.putChild(b"game", fe_game)
+
+        self.logger.info(f"Ready on port {self.config.frontend.port} serving {len(fe_game.children)} games")
+
+    def render_GET(self, request):
+        self.logger.debug(f"{request.getClientIP()} -> {request.uri.decode()}")
+        template = self.environment.get_template("core/frontend/index.jinja")        
+        return template.render(server_name=self.config.server.name, title=self.config.server.name, game_list=self.game_list).encode("utf-16")
+
+class FE_Base(resource.Resource):
+    """
+    A Generic skeleton class that all frontend handlers should inherit from
+    Initializes the environment, data, logger, config, and sets isLeaf to true
+    It is expected that game implementations of this class overwrite many of these
+    """
+    isLeaf = True    
+    def __init__(self, cfg: CoreConfig, environment: jinja2.Environment) -> None:
+        self.core_config = cfg
+        self.data = Data(cfg)
+        self.logger = logging.getLogger('frontend')
+        self.environment = environment
+        self.nav_name = "nav_name"
+
+class FE_Gate(FE_Base):
+    def render_GET(self, request: Request):        
+        self.logger.debug(f"{request.getClientIP()} -> {request.uri.decode()}")
+        uri: str = request.uri.decode()
+        if uri.startswith("/gate/create"):
+            return self.create_user(request)
+
+        if b'e' in request.args:
+            try:
+                err = int(request.args[b'e'][0].decode())
+            except:
+                err = 0
+
+        else: err = 0
+
+        template = self.environment.get_template("core/frontend/gate/gate.jinja")        
+        return template.render(title=f"{self.core_config.server.name} | Login Gate", error=err).encode("utf-16")
+    
+    def render_POST(self, request: Request):
+        uri = request.uri.decode()
+        ip = request.getClientAddress().host
+        
+        if uri == "/gate/gate.login":            
+            access_code: str = request.args[b"access_code"][0].decode()
+            passwd: str = request.args[b"passwd"][0]
+            if passwd == b"":
+                passwd = None
+
+            uid = self.data.card.get_user_id_from_card(access_code)
+            if uid is None:
+                return redirectTo(b"/gate?e=1", request)
+                        
+            if passwd is None:
+                sesh = self.data.user.login(uid, ip=ip)
+
+                if sesh is not None:
+                    return redirectTo(f"/gate/create?ac={access_code}".encode(), request)
+                return redirectTo(b"/gate?e=1", request)
+
+            salt = bcrypt.gensalt()
+            hashed = bcrypt.hashpw(passwd, salt)
+            sesh = self.data.user.login(uid, hashed, ip)
+
+            if sesh is None:
+                return redirectTo(b"/gate?e=1", request)
+
+            request.addCookie('session', sesh)            
+            return redirectTo(b"/user", request)
+        
+        elif uri == "/gate/gate.create":
+            access_code: str = request.args[b"access_code"][0].decode()
+            username: str = request.args[b"username"][0]
+            email: str = request.args[b"email"][0].decode()
+            passwd: str = request.args[b"passwd"][0]
+
+            uid = self.data.card.get_user_id_from_card(access_code)
+            if uid is None:
+                return redirectTo(b"/gate?e=1", request)
+
+            salt = bcrypt.gensalt()
+            hashed = bcrypt.hashpw(passwd, salt)
+
+            result = self.data.user.create_user(uid, username, email, hashed.decode(), 1)
+            if result is None:
+                return redirectTo(b"/gate?e=3", request)
+            
+            sesh = self.data.user.login(uid, hashed, ip)
+            if sesh is None:
+                return redirectTo(b"/gate", request)
+            request.addCookie('session', sesh)
+            
+            return redirectTo(b"/user", request)
+
+        else:
+            return b""
+
+    def create_user(self, request: Request):
+        if b'ac' not in request.args or len(request.args[b'ac'][0].decode()) != 20:            
+            return redirectTo(b"/gate?e=2", request)
+
+        ac = request.args[b'ac'][0].decode()
+        
+        template = self.environment.get_template("core/frontend/gate/create.jinja")        
+        return template.render(title=f"{self.core_config.server.name} | Create User", code=ac).encode("utf-16")
+
+class FE_User(FE_Base):
+    def render_GET(self, request: Request):
+        template = self.environment.get_template("core/frontend/user/index.jinja")
+        return template.render().encode("utf-16")
+        if b'session' not in request.cookies:
+            return redirectTo(b"/gate", request)
+
+class FE_Game(FE_Base):
+    isLeaf = False
+    children: Dict[str, Any] = {}
+
+    def getChild(self, name: bytes, request: Request):
+        if name == b'':
+            return self
+        return resource.Resource.getChild(self, name, request)
+
+    def render_GET(self, request: Request) -> bytes:
+        return redirectTo(b"/user", request)

core/frontend/gate/create.jinja

@@ -0,0 +1,24 @@
+{% extends "core/frontend/index.jinja" %}
+{% block content %}
+<h1>Create User</h1>
+<form id="create" style="max-width: 240px; min-width: 10%;" action="/gate/gate.create" method="post">
+    <div class="form-group row">
+        <label for="access_code">Card Access Code</label><br>
+        <input class="form-control" name="access_code" id="access_code" type="text" placeholder="00000000000000000000" value={{ code }} maxlength="20" readonly>
+    </div>
+    <div class="form-group row">
+        <label for="username">Username</label><br>
+        <input id="username" class="form-control" name="username" type="text" placeholder="username">
+    </div>
+    <div class="form-group row">
+        <label for="email">Email</label><br>
+        <input id="email" class="form-control" name="email" type="email" placeholder="example@example.com">
+    </div>
+    <div class="form-group row">
+        <label for="passwd">Password</label><br>
+        <input id="passwd" class="form-control" name="passwd" type="password" placeholder="password">
+    </div>
+    <p></p>
+    <input id="submit" class="btn btn-primary" style="display: block; margin: 0 auto;" type="submit" value="Create">
+</form>
+{% endblock content %}

core/frontend/gate/gate.jinja

@@ -0,0 +1,17 @@
+{% extends "core/frontend/index.jinja" %}
+{% block content %}
+<h1>Gate</h1>
+{% include "core/frontend/widgets/err_banner.jinja" %}
+<form id="login" style="max-width: 240px; min-width: 10%;" action="/gate/gate.login" method="post">
+    <div class="form-group row">
+        <label for="access_code">Card Access Code</label><br>
+        <input form="login" class="form-control" name="access_code" id="access_code" type="text" placeholder="00000000000000000000" maxlength="20" required>
+    </div>
+    <div class="form-group row">
+        <label for="passwd">Password</label><br>
+        <input id="passwd" class="form-control" name="passwd" type="password" placeholder="password">
+    </div>
+    <p></p>
+    <input id="submit" class="btn btn-primary" style="display: block; margin: 0 auto;" form="login" type="submit" value="Login">
+</form>
+{% endblock content %}

core/frontend/index.jinja

@@ -0,0 +1,88 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>{{ title }}</title>
+        <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65" crossorigin="anonymous">
+        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-kenU1KFdBIe4zVF0s0G1M5b4hcpxyD9F7jL+jjXkk+Q2h455rYXK/7HAuoJl+0I4" crossorigin="anonymous"></script>
+        <style>
+            html {
+                background-color: #181a1b !important;
+                margin: 10px;
+            }
+            html {
+                color-scheme: dark !important;
+            }
+            html, body, input, textarea, select, button, dialog {
+                background-color: #181a1b;
+            }
+            html, body, input, textarea, select, button {
+                border-color: #736b5e;
+                color: #e8e6e3;
+            }
+            a {
+                color: #3391ff;
+            }
+            table {
+                border-color: #545b5e;
+            }
+            ::placeholder {
+                color: #b2aba1;
+            }
+            input:-webkit-autofill,
+            textarea:-webkit-autofill,
+            select:-webkit-autofill {
+                background-color: #404400 !important;
+                color: #e8e6e3 !important;
+            }
+            ::-webkit-scrollbar {
+                background-color: #202324;
+                color: #aba499;
+            }
+            ::-webkit-scrollbar-thumb {
+                background-color: #454a4d;
+            }
+            ::-webkit-scrollbar-thumb:hover {
+                background-color: #575e62;
+            }
+            ::-webkit-scrollbar-thumb:active {
+                background-color: #484e51;
+            }
+            ::-webkit-scrollbar-corner {
+                background-color: #181a1b;
+            }
+            * {
+                scrollbar-color: #454a4d #202324;
+            }
+            ::selection {
+                background-color: #004daa !important;
+                color: #e8e6e3 !important;
+            }
+            ::-moz-selection {
+                background-color: #004daa !important;
+                color: #e8e6e3 !important;
+            }
+            input[type="text"], input[type="text"]:focus, input[type="password"], input[type="password"]:focus, input[type="email"], input[type="email"]:focus {
+                background-color: #202324 !important;
+                color: #e8e6e3;
+            }
+            form {
+                outline: 1px solid grey; 
+                padding: 20px;
+                padding-top: 10px;
+                padding-bottom: 10px;
+            }
+            .err-banner {
+                background-color: #AA0000;
+                padding: 20px;
+                margin-bottom: 10px;
+                width: 15%;
+            }
+        </style>
+    </head>
+    <body>
+    {% include "core/frontend/widgets/topbar.jinja" %}
+    {% block content %}
+    <h1>{{ server_name }}</h1>
+    {% endblock content %}
+    </body>
+</html>

core/frontend/user/index.jinja

@@ -0,0 +1,4 @@
+{% extends "core/frontend/index.jinja" %}
+{% block content %}
+<h1>testing</h1>
+{% endblock content %}

core/frontend/widgets/err_banner.jinja

@@ -0,0 +1,14 @@
+{% if error > 0 %}
+<div class="err-banner">
+<h3>Error</h3>
+{% if error == 1 %}
+Card not registered, or wrong password
+{% elif error == 2 %}
+Missing or malformed access code
+{% elif error == 3 %}
+Failed to create user
+{% else %}
+An unknown error occoured
+{% endif %}
+</div>
+{% endif %}

core/frontend/widgets/topbar.jinja

@@ -0,0 +1,13 @@
+<div style="background: #333; color: #f9f9f9; width: 10%; height: 50px; line-height: 50px; text-align: center; float: left;">
+    Navigation
+</div>
+<div style="background: #333; color: #f9f9f9; width: 80%; height: 50px; line-height: 50px; padding-left: 10px; float: left;">
+    <a href=/><button class="btn btn-primary">Home</button></a>&nbsp;
+    {% for game in game_list %}    
+    <a href=game/{{ game.url }}><button class="btn btn-success">{{ game.name }}</button></a>&nbsp;
+    {% endfor %}
+</div>
+</div>
+<div style="background: #333; color: #f9f9f9; width: 10%; height: 50px; line-height: 50px; text-align: center; float: left;">
+    <a href="/gate"><button class="btn btn-primary">Gate</button></a>
+</div>

core/mucha.py

@@ -35,9 +35,14 @@ class MuchaServlet:
             return b""
 
         req = MuchaAuthRequest(req_dict)
-        self.logger.info(f"Mucha request {vars(req)}")
-        resp = MuchaAuthResponse(mucha_url=f"{self.config.mucha.hostname}:{self.config.mucha.port}")
-        self.logger.info(f"Mucha response {vars(resp)}")
+        self.logger.debug(f"Mucha request {vars(req)}")
+
+        if self.config.server.is_develop:
+            resp = MuchaAuthResponse(mucha_url=f"{self.config.mucha.hostname}:{self.config.mucha.port}")
+        else:
+            resp = MuchaAuthResponse(mucha_url=f"{self.config.mucha.hostname}")
+
+        self.logger.debug(f"Mucha response {vars(resp)}")
 
         return self.mucha_postprocess(vars(resp))
     
@@ -48,9 +53,14 @@ class MuchaServlet:
             return b""
 
         req = MuchaUpdateRequest(req_dict)
-        self.logger.info(f"Mucha request {vars(req)}")
-        resp = MuchaUpdateResponse(mucha_url=f"{self.config.mucha.hostname}:{self.config.mucha.port}")
-        self.logger.info(f"Mucha response {vars(resp)}")
+        self.logger.debug(f"Mucha request {vars(req)}")
+
+        if self.config.server.is_develop:
+            resp = MuchaUpdateResponse(mucha_url=f"{self.config.mucha.hostname}:{self.config.mucha.port}")
+        else:
+            resp = MuchaUpdateResponse(mucha_url=f"{self.config.mucha.hostname}")
+
+        self.logger.debug(f"Mucha response {vars(resp)}")
 
         return self.mucha_postprocess(vars(resp))
 

core/title.py

@@ -71,4 +71,4 @@ class TitleServlet():
             self.logger.warn(f"{code} does not dispatch POST")
             return b""
 
-        return index.render_POST(request, endpoints["version"], endpoints["endpoint"])
+        return index.render_POST(request, int(endpoints["version"]), endpoints["endpoint"])

core/utils.py

@@ -1,6 +1,6 @@
-from typing import Dict, List, Any, Optional
+from typing import Dict, Any
 from types import ModuleType
-import zlib, base64
+import logging
 import importlib
 from os import walk
 
@@ -17,6 +17,6 @@ class Utils:
                         ret[dir] = mod
 
                     except ImportError as e:
-                        print(f"{dir} - {e}")
+                        logging.getLogger("core").error(f"get_all_titles: {dir} - {e}")
                         raise
             return ret

dbutils.py

@@ -23,22 +23,22 @@ if __name__=='__main__':
 
     elif args.action == "upgrade" or args.action == "rollback":
         if args.version is None:
-            print("Must set game and version to migrate to")
+            data.logger.error("Must set game and version to migrate to")
             exit(0)
 
         if args.game is None:
-            print("No game set, upgrading core schema")
+            data.logger.info("No game set, upgrading core schema")
             data.migrate_database("CORE", int(args.version))
 
         else:
             data.migrate_database(args.game, int(args.version), args.action)
     
     elif args.action == "migrate":
-        print("Migrating from old schema to new schema")
+        data.logger.info("Migrating from old schema to new schema")
         data.restore_from_old_schema()
     
     elif args.action == "dump":
-        print("Dumping old schema to migrate to new schema")
+        data.logger.info("Dumping old schema to migrate to new schema")
         data.dump_db()
     
     elif args.action == "generate":

docs/INSTALL_UBUNTU.md

@@ -0,0 +1,129 @@
+# ARTEMiS - Ubuntu 20.04 LTS Guide
+This step-by-step guide assumes that you are using a fresh install of Ubuntu 20.04 LTS, some of the steps can be skipped if you already have an installation with MySQL 5.7 or even some of the modules already present on your environment
+
+# Setup
+## Install memcached module
+1. sudo apt-get install memcached
+2. Under the file /etc/memcached.conf, please make sure the following parameters are set:
+
+```
+# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default
+# Note that the daemon will grow to this size, but does not start out holding this much
+# memory
+
+-I 128m
+-m 1024
+```
+
+** This is mandatory to avoid memcached overload caused by Crossbeats or by massive profiles
+
+3. Restart memcached using:  sudo systemctl restart memcached
+
+## Install MySQL 5.7
+```
+sudo apt update
+sudo apt install wget -y
+wget https://dev.mysql.com/get/mysql-apt-config_0.8.12-1_all.deb
+sudo dpkg -i mysql-apt-config_0.8.12-1_all.deb
+```
+    1. During the first prompt, select Ubuntu Bionic
+    2. Select the default option
+    3. Select MySQL 5.7
+    4. Select the last option
+```
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29
+sudo apt-get update
+sudo apt-cache policy mysql-server
+sudo apt install -f mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7*
+```
+
+## Default Configuration for MySQL Server
+1. sudo mysql_secure_installation
+> Make sure to follow the steps that will be prompted such as changing the mysql root password and such
+
+2. Test your MySQL Server login by doing the following command :
+> mysql -u root -p
+
+## Create the default ARTEMiS database and user
+1. mysql -u root -p
+2. Please change the password indicated in the next line for a custom secure one and continue with the next commands
+
+```
+CREATE USER 'aime'@'localhost' IDENTIFIED BY 'MyStrongPass.';
+CREATE DATABASE aime;
+GRANT Alter,Create,Delete,Drop,Index,Insert,References,Select,Update ON aime.* TO 'aime'@'localhost';
+FLUSH PRIVILEGES;
+exit;
+```
+
+3. sudo systemctl restart mysql
+
+## Install Python modules
+```
+sudo apt-get install python3-dev default-libmysqlclient-dev build-essential mysql-client libmysqlclient-dev libmemcached-dev
+sudo apt install libpython3.8-dev
+sudo apt-get install python3-software-properties
+sudo apt install python3-pip
+sudo pip3 install --upgrade pip testresources
+sudo pip3 install --upgrade pip setuptools
+sudo apt-get install python3-tk
+```
+7. Change your work path to the ARTEMiS root folder using 'cd' and install the requirements:
+> sudo python3 -m pip install -r requirements.txt
+
+## Copy/Rename the folder example_config to config
+
+## Adjust /config/core.yaml
+1. Make sure to change the server listen_address to be set to your local machine IP (ex.: 192.168.1.xxx) 
+2. Adjust the proper MySQL information you created earlier
+3. Add the AimeDB key at the bottom of the file
+
+## Create the database tables for ARTEMiS
+1. sudo python3 dbutils.py create
+
+2. If you get "No module named Crypto", run the following command:
+```
+sudo pip uninstall crypto
+sudo pip uninstall pycrypto
+sudo pip install pycrypto
+```
+
+## Firewall Adjustements 
+```
+sudo ufw allow 80
+sudo ufw allow 443
+sudo ufw allow 8443
+sudo ufw allow 22345
+sudo ufw allow 8090
+sudo ufw allow 8444
+sudo ufw allow 9000
+```
+
+## Running the ARTEMiS instance
+1. sudo python3 index.py
+
+# Troubleshooting
+
+## Game does not connect to ARTEMiS Allnet server
+1. Double-check your core.yaml, the listen_address is most likely either not binded to the proper IP or the port is not opened
+
+## Game does not connect to Title Server
+1. Verify that your core.yaml is setup properly for both the server listen_address and title hostname
+2. Boot your game and verify that an AllNet response does show and if it does, attempt to open the URI that is shown under a browser such as Edge, Chrome & Firefox.
+3. If a page is shown, the server is working properly and if it doesn't, double check your port forwarding and also that you have entered the proper local IP under the Title hostname in core.yaml.
+
+## Unhandled command under AimeDB
+1. Double check your AimeDB key under core.yaml, it is incorrect.
+
+## Memcache failed, error 3
+1. Make sure memcached is properly installed and running. You can check the status of the service using the following command:
+> sudo systemctl status memcached
+2. If it is failing, double check the /etc/memcached.conf file, it may have duplicated arguments like the -I and -m
+3. If it is still not working afterward, you can proceed with a workaround by manually editing the /core/data/cache.py file.
+```
+# Make memcache optional
+try:
+    has_mc = False
+except ModuleNotFoundError:
+    has_mc = False
+```

docs/INSTALL_WINDOWS.md

@@ -0,0 +1,83 @@
+# ARTEMiS - Windows 10/11 Guide
+This step-by-step guide assumes that you are using a fresh install of Windows 10/11 without MySQL installed, some of the steps can be skipped if you already have an installation with MySQL 8.0 or even some of the modules already present on your environment
+
+# Setup
+## Install Python Python 3.9 (recommended) or 3.10
+1. Download Python 3.9 : [Link](https://www.python.org/ftp/python/3.9.13/python-3.9.13-amd64.exe)
+2. Install python-3.9.13-amd64.exe
+    1. Select Customize installation
+    2. Make sure that pip, tcl/tk, and the for all users are checked and hit Next
+    3. Make sure that you enable "Create shortcuts for installed applications" and "Add Python to environment variables" and hit Install
+
+## Install MySQL 8.0
+1. Download MySQL 8.0 Server : [Link](https://cdn.mysql.com//Downloads/MySQLInstaller/mysql-installer-web-community-8.0.31.0.msi)
+2. Install mysql-installer-web-community-8.0.31.0.msi
+    1. Click on "Add ..." on the side
+    2. Click on the "+" next to MySQL Servers
+    3. Make sure MySQL Server 8.0.29 - X64 is under the products to be installed.
+    4. Hit Next and Next once installed
+    5. Select the configuration type "Development Computer"
+    6. Hit Next
+    7. Select "Use Legacy Authentication Method (Retain MySQL 5.x compatibility)" and hit Next
+    8. Enter a root password and then hit Next >
+    9. Leave everything under Windows Service as default and hit Next >
+    10. Click on Execute and for it to finish and hit Next> and then Finish
+3. Open MySQL 8.0 Command Line Client and login as your root user
+4. Type those commands to create your user and the database
+```
+CREATE USER 'aime'@'localhost' IDENTIFIED BY 'MyStrongPass.';
+CREATE DATABASE aime;
+GRANT Alter,Create,Delete,Drop,Index,Insert,References,Select,Update ON aime.* TO 'aime'@'localhost';
+FLUSH PRIVILEGES;
+exit;
+```
+
+## Install Python modules
+1. Change your work path to the artemis-master folder using 'cd' and install the requirements:
+> pip install -r requirements.txt
+
+## Copy/Rename the folder example_config to config
+
+## Adjust /config/core.yaml
+
+1. Make sure to change the server listen_address to be set to your local machine IP (ex.: 192.168.1.xxx) 
+    - In case you want to run this only locally, set the following values:
+```
+server:
+    listen_address: 0.0.0.0
+title: 
+    hostname: localhost
+```
+2. Adjust the proper MySQL information you created earlier
+3. Add the AimeDB key at the bottom of the file
+4. If the webui is needed, change the flag from False to True
+
+## Create the database tables for ARTEMiS
+> python dbutils.py create
+
+## Firewall Adjustements 
+Make sure the following ports are open both on your router and local Windows firewall in case you want to use this for public use (NOT recommended):
+> Port 80 (TCP), 443 (TCP), 8443 (TCP), 22345 (TCP), 8090 (TCP) **webui, 8444 (TCP) **mucha, 9000 (TCP)
+
+## Running the ARTEMiS instance
+> python index.py
+
+# Troubleshooting
+
+## Game does not connect to ARTEMiS Allnet server
+1. Double-check your core.yaml, the listen_address is most likely either not binded to the proper IP or the port is not opened
+
+## Game does not connect to Title Server
+1. Verify that your core.yaml is setup properly for both the server listen_address and title hostname
+2. Boot your game and verify that an AllNet response does show and if it does, attempt to open the URI that is shown under a browser such as Edge, Chrome & Firefox.
+3. If a page is shown, the server is working properly and if it doesn't, double check your port forwarding and also that you have entered the proper local IP under the Title hostname in core.yaml.
+
+## Unhandled command under AimeDB
+1. Double check your AimeDB key under core.yaml, it is incorrect.
+
+## AttributeError: module 'collections' has no attribute 'Hashable'
+1. This means the pyYAML module is obsolete, simply rerun pip with the -U (force update) flag, as shown below.
+    - Change your work path to the artemis-master (or artemis-develop) folder using 'cd' and run the following commands:
+```
+pip install -r requirements.txt -U
+```

example_config/diva.yaml

@@ -2,3 +2,6 @@ server:
   enable: True
   loglevel: "info"
 
+mods:
+  unlock_all_modules: True
+  unlock_all_items: True

example_config/nginx_example.conf

@@ -18,7 +18,7 @@ server {
 	}
 }
 
-# SSL titles
+# SSL titles, comment out if you don't plan on accepting SSL titles
 server {
 	listen 443 ssl default_server;
 	listen [::]:443 ssl default_server;	
@@ -57,4 +57,99 @@ server {
 	location / {
 		proxy_pass http://localhost:8444/;
 	}
+}
+
+# Pokken, comment this out if you don't plan on serving pokken.
+server {
+	listen 443 ssl;	
+	server_name pokken.hostname.here;
+	
+	ssl_certificate /path/to/cert/pokken.pem;
+	ssl_certificate_key /path/to/cert/pokken.key;
+	ssl_session_timeout 1d;
+	ssl_session_cache shared:MozSSL:10m;
+	ssl_session_tickets off;
+
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
+	ssl_ciphers "ALL:@SECLEVEL=1";
+	ssl_prefer_server_ciphers off;
+
+	location / {
+		proxy_pass http://localhost:8080/;
+	}
+}
+
+# CXB, comment this out if you don't plan on serving crossbeats.
+server {
+	listen 443 ssl;	
+	server_name cxb.hostname.here;
+	
+	ssl_certificate /path/to/cert/cxb.pem;
+	ssl_certificate_key /path/to/cert/cxb.key;
+	ssl_session_timeout 1d;
+	ssl_session_cache shared:MozSSL:10m;
+	ssl_session_tickets off;
+
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
+	ssl_ciphers "ALL:@SECLEVEL=1";
+	ssl_prefer_server_ciphers off;
+
+	location / {
+		proxy_pass http://localhost:8080/SDBT/104/;
+	}
+}
+
+# CXB, comment this out if you don't plan on serving crossbeats.
+server {
+	listen 443 ssl;	
+	server_name cxb.hostname.here;
+	
+	ssl_certificate /path/to/cert/cxb.pem;
+	ssl_certificate_key /path/to/cert/cxb.key;
+	ssl_session_timeout 1d;
+	ssl_session_cache shared:MozSSL:10m;
+	ssl_session_tickets off;
+
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
+	ssl_ciphers "ALL:@SECLEVEL=1";
+	ssl_prefer_server_ciphers off;
+
+	location / {
+		proxy_pass http://localhost:8080/SDBT/104/;
+	}
+}
+
+# Frontend, set to redirect to HTTPS. Comment out if you don't intend to use the frontend
+server {
+    listen 80;
+	server_name frontend.hostname.here
+
+    location / {
+        return 301 https://$host$request_uri;
+		# If you don't want https redirection, comment the line above and uncomment the line below
+		# proxy_pass http://localhost:8090/;
+    }
+}
+
+# Frontend HTTPS. Comment out if you on't intend to use the frontend
+server {
+    listen 443 ssl;
+
+    ssl_certificate /path/to/cert/frontend.pem;
+	ssl_certificate_key /path/to/cert/frontend.key;
+    ssl_session_timeout 1d;
+    ssl_session_cache shared:MozSSL:10m;  # about 40000 sessions
+    ssl_session_tickets off;
+
+    # intermediate configuration
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+    ssl_prefer_server_ciphers off;
+
+    # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+    add_header Strict-Transport-Security "max-age=63072000" always;
+
+	location / {
+		proxy_pass http://localhost:8090/;
+	}
 }

example_config/pokken.yaml

@@ -1,8 +1,6 @@
 server:
   enable: True
   loglevel: "info"
-  hostname: "localhost"
-  ssl_enable: False
   port: 9000
   port_matching: 9001
   ssl_cert: cert/pokken.crt

example_config/wacca.yaml

@@ -1,6 +1,7 @@
 server:
   enable: True
   loglevel: "info"
+  prefecture_name: "Hokkaido"
 
 mods:
   always_vip: True

index.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 import argparse
+import logging, coloredlogs
+from logging.handlers import TimedRotatingFileHandler
 from typing import Dict
 import yaml
 from os import path, mkdir, access, W_OK
@@ -17,11 +19,13 @@ class HttpDispatcher(resource.Resource):
         self.isLeaf = True
         self.map_get = Mapper()
         self.map_post = Mapper()
+        self.logger = logging.getLogger("core")
         
         self.allnet = AllnetServlet(cfg, config_dir)
         self.title = TitleServlet(cfg, config_dir)
         self.mucha = MuchaServlet(cfg)
 
+        self.map_post.connect('allnet_ping', '/naomitest.html', controller="allnet", action='handle_naomitest', conditions=dict(method=['GET']))
         self.map_post.connect('allnet_poweron', '/sys/servlet/PowerOn', controller="allnet", action='handle_poweron', conditions=dict(method=['POST']))
         self.map_post.connect('allnet_downloadorder', '/sys/servlet/DownloadOrder', controller="allnet", action='handle_dlorder', conditions=dict(method=['POST']))
         self.map_post.connect('allnet_billing', '/request', controller="allnet", action='handle_billing_request', conditions=dict(method=['POST']))
@@ -29,31 +33,39 @@ class HttpDispatcher(resource.Resource):
         self.map_post.connect('mucha_boardauth', '/mucha/boardauth.do', controller="mucha", action='handle_boardauth', conditions=dict(method=['POST']))
         self.map_post.connect('mucha_updatacheck', '/mucha/updatacheck.do', controller="mucha", action='handle_updatacheck', conditions=dict(method=['POST']))
 
-        self.map_get.connect("title_get", "/{game}/{version}/{endpoint:.*?}", controller="title", action="render_GET", requirements=dict(game=R"S..."))
-        self.map_post.connect("title_post", "/{game}/{version}/{endpoint:.*?}", controller="title", action="render_POST", requirements=dict(game=R"S..."))
+        self.map_get.connect("title_get", "/{game}/{version}/{endpoint:.*?}", controller="title", action="render_GET", conditions=dict(method=['GET']), requirements=dict(game=R"S..."))
+        self.map_post.connect("title_post", "/{game}/{version}/{endpoint:.*?}", controller="title", action="render_POST", conditions=dict(method=['POST']), requirements=dict(game=R"S..."))
 
-    def render_POST(self, request: Request) -> bytes:    
+    def render_GET(self, request: Request) -> bytes:    
         test = self.map_get.match(request.uri.decode())
         if test is None:
-            return b""
+            self.logger.debug(f"Unknown GET endpoint {request.uri.decode()} from {request.getClientAddress().host} to port {request.getHost().port}")
+            request.setResponseCode(404)
+            return b"Endpoint not found."
 
         return self.dispatch(test, request)
 
     def render_POST(self, request: Request) -> bytes:    
         test = self.map_post.match(request.uri.decode())
         if test is None:
-            return b""
+            self.logger.debug(f"Unknown POST endpoint {request.uri.decode()} from {request.getClientAddress().host} to port {request.getHost().port}")
+            request.setResponseCode(404)
+            return b"Endpoint not found."
         
         return self.dispatch(test, request)
 
     def dispatch(self, matcher: Dict, request: Request) -> bytes:
         controller = getattr(self, matcher["controller"], None)
         if controller is None:
-            return b""
+            self.logger.error(f"Controller {matcher['controller']} not found via endpoint {request.uri.decode()}")
+            request.setResponseCode(404)
+            return b"Endpoint not found."
         
         handler = getattr(controller, matcher["action"], None)
         if handler is None:
-            return b""
+            self.logger.error(f"Action {matcher['action']} not found in controller {matcher['controller']} via endpoint {request.uri.decode()}")
+            request.setResponseCode(404)
+            return b"Endpoint not found."
         
         url_vars = matcher
         url_vars.pop("controller")
@@ -79,22 +91,40 @@ if __name__ == "__main__":
     cfg: CoreConfig = CoreConfig()
     cfg.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
 
+    logger = logging.getLogger("core")
+    log_fmt_str = "[%(asctime)s] Core | %(levelname)s | %(message)s"
+    log_fmt = logging.Formatter(log_fmt_str)        
+
+    fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(cfg.server.log_dir, "core"), when="d", backupCount=10)
+    fileHandler.setFormatter(log_fmt)
+    
+    consoleHandler = logging.StreamHandler()
+    consoleHandler.setFormatter(log_fmt)
+
+    logger.addHandler(fileHandler)
+    logger.addHandler(consoleHandler)
+    
+    log_lv = logging.DEBUG if cfg.server.is_develop else logging.INFO
+    logger.setLevel(log_lv)
+    coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
+
     if not path.exists(cfg.server.log_dir):
         mkdir(cfg.server.log_dir)
     
     if not access(cfg.server.log_dir, W_OK):
-        print(f"Log directory {cfg.server.log_dir} NOT writable, please check permissions")
+        logger.error(f"Log directory {cfg.server.log_dir} NOT writable, please check permissions")
         exit(1)
 
     if not cfg.aimedb.key:
-        print("!!AIMEDB KEY BLANK, SET KEY IN CORE.YAML!!")
+        logger.error("!!AIMEDB KEY BLANK, SET KEY IN CORE.YAML!!")
         exit(1)
     
-    print(f"ARTEMiS starting in {'develop' if cfg.server.is_develop else 'production'} mode")
+    logger.info(f"ARTEMiS starting in {'develop' if cfg.server.is_develop else 'production'} mode")
 
     allnet_server_str = f"tcp:{cfg.allnet.port}:interface={cfg.server.listen_address}"    
     title_server_str = f"tcp:{cfg.title.port}:interface={cfg.server.listen_address}"
     adb_server_str = f"tcp:{cfg.aimedb.port}:interface={cfg.server.listen_address}"
+    frontend_server_str = f"tcp:{cfg.frontend.port}:interface={cfg.server.listen_address}"
 
     billing_server_str = f"tcp:{cfg.billing.port}:interface={cfg.server.listen_address}"
     if cfg.server.is_develop:
@@ -106,6 +136,9 @@ if __name__ == "__main__":
     endpoints.serverFromString(reactor, allnet_server_str).listen(server.Site(dispatcher))
     endpoints.serverFromString(reactor, adb_server_str).listen(AimedbFactory(cfg))
 
+    if cfg.frontend.enable:
+        endpoints.serverFromString(reactor, frontend_server_str).listen(server.Site(FrontendServlet(cfg, args.config)))
+
     if cfg.billing.port > 0:
         endpoints.serverFromString(reactor, billing_server_str).listen(server.Site(dispatcher))
     

read.py

@@ -85,7 +85,7 @@ if __name__ == "__main__":
     log_fmt = logging.Formatter(log_fmt_str)
     logger = logging.getLogger("reader")
 
-    fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(config.server.logs, "reader"), when="d", backupCount=10)
+    fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(config.server.log_dir, "reader"), when="d", backupCount=10)
     fileHandler.setFormatter(log_fmt)
     
     consoleHandler = logging.StreamHandler()
@@ -94,8 +94,9 @@ if __name__ == "__main__":
     logger.addHandler(fileHandler)
     logger.addHandler(consoleHandler)
     
-    logger.setLevel(logging.INFO)
-    coloredlogs.install(level=logging.INFO, logger=logger, fmt=log_fmt_str)
+    log_lv = logging.DEBUG if config.server.is_develop else logging.INFO
+    logger.setLevel(log_lv)
+    coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
 
     if args.series is None or args.version is None:
         logger.error("Game or version not specified")

requirements.txt

@@ -13,4 +13,6 @@ coloredlogs
 pylibmc; platform_system != "Windows"
 wacky
 Routes
+bcrypt
+jinja2
 protobuf

titles/chuni/new.py

@@ -47,10 +47,10 @@ class ChuniNew(ChuniBase):
                 "matchErrorLimit": 9999,
                 "romVersion": "2.00.00",
                 "dataVersion": "2.00.00",
-                "matchingUri": f"http://{self.core_cfg.server.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
-                "matchingUriX": f"http://{self.core_cfg.server.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
-                "udpHolePunchUri": f"http://{self.core_cfg.server.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
-                "reflectorUri": f"http://{self.core_cfg.server.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
+                "matchingUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
+                "matchingUriX": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
+                "udpHolePunchUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
+                "reflectorUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
             },
                 "isDumpUpload": "false",
                 "isAou": "false",

titles/chuni/newplus.py

@@ -16,8 +16,8 @@ class ChuniNewPlus(ChuniNew):
         ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["romVersion"] = "2.05.00"
         ret["gameSetting"]["dataVersion"] = "2.05.00"
-        ret["gameSetting"]["matchingUri"] = f"http://{self.core_cfg.server.hostname}:{self.core_cfg.title.port}/SDHD/205/ChuniServlet/"
-        ret["gameSetting"]["matchingUriX"] = f"http://{self.core_cfg.server.hostname}:{self.core_cfg.title.port}/SDHD/205/ChuniServlet/"
-        ret["gameSetting"]["udpHolePunchUri"] = f"http://{self.core_cfg.server.hostname}:{self.core_cfg.title.port}/SDHD/205/ChuniServlet/"
-        ret["gameSetting"]["reflectorUri"] = f"http://{self.core_cfg.server.hostname}:{self.core_cfg.title.port}/SDHD/205/ChuniServlet/"
+        ret["gameSetting"]["matchingUri"] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/205/ChuniServlet/"
+        ret["gameSetting"]["matchingUriX"] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/205/ChuniServlet/"
+        ret["gameSetting"]["udpHolePunchUri"] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/205/ChuniServlet/"
+        ret["gameSetting"]["reflectorUri"] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/205/ChuniServlet/"
         return ret

titles/chuni/read.py

@@ -82,7 +82,7 @@ class ChuniReader(BaseReader):
                         for MusicFumenData in fumens.findall('MusicFumenData'):
                             fumen_path = MusicFumenData.find('file').find("path")
                             
-                            if fumen_path.text is not None:
+                            if fumen_path is not None:
                                 chart_id = MusicFumenData.find('type').find('id').text
                                 if chart_id == "4":
                                     level = float(xml_root.find("starDifType").text)

titles/chuni/schema/score.py

@@ -29,6 +29,9 @@ course = Table(
     Column("param3", Integer),
     Column("param4", Integer),
     Column("isClear", Boolean),
+    Column("theoryCount", Integer),
+    Column("orderId", Integer),
+    Column("playerRating", Integer),
     UniqueConstraint("user", "courseId", name="chuni_score_course_uk"),
     mysql_charset='utf8mb4'
 )

titles/chuni/schema/static.py

@@ -34,7 +34,7 @@ music = Table(
     Column("level", Float),
     Column("genre", String(255)),
     Column("jacketPath", String(255)),
-    Column("worldsEndTag", String(20)),
+    Column("worldsEndTag", String(7)),
     UniqueConstraint("version", "songId", "chartId", name="chuni_static_music_uk"),
     mysql_charset='utf8mb4'
 )

titles/diva/config.py

@@ -1,17 +1,33 @@
 from core.config import CoreConfig
 
+
 class DivaServerConfig():
     def __init__(self, parent_config: "DivaConfig") -> None:
         self.__config = parent_config
-    
+
     @property
     def enable(self) -> bool:
         return CoreConfig.get_config_field(self.__config, 'diva', 'server', 'enable', default=True)
-    
+
     @property
     def loglevel(self) -> int:
         return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'diva', 'server', 'loglevel', default="info"))
 
+
+class DivaModsConfig():
+    def __init__(self, parent_config: "DivaConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def unlock_all_modules(self) -> bool:
+        return CoreConfig.get_config_field(self.__config, 'diva', 'mods', 'unlock_all_modules', default=True)
+
+    @property
+    def unlock_all_items(self) -> bool:
+        return CoreConfig.get_config_field(self.__config, 'diva', 'mods', 'unlock_all_items', default=True)
+
+
 class DivaConfig(dict):
     def __init__(self) -> None:
-        self.server = DivaServerConfig(self)
+        self.server = DivaServerConfig(self)
+        self.mods = DivaModsConfig(self)

titles/diva/database.py

@@ -1,6 +1,7 @@
 from core.data import Data
 from core.config import CoreConfig
-from titles.diva.schema import DivaProfileData, DivaScoreData, DivaItemData, DivaStaticData
+from titles.diva.schema import DivaProfileData, DivaScoreData, DivaModuleData, DivaCustomizeItemData, DivaPvCustomizeData, DivaItemData, DivaStaticData
+
 
 class DivaData(Data):
     def __init__(self, cfg: CoreConfig) -> None:
@@ -8,5 +9,8 @@ class DivaData(Data):
 
         self.profile = DivaProfileData(self.config, self.session)
         self.score = DivaScoreData(self.config, self.session)
+        self.module = DivaModuleData(self.config, self.session)
+        self.customize = DivaCustomizeItemData(self.config, self.session)
+        self.pv_customize = DivaPvCustomizeData(self.config, self.session)
         self.item = DivaItemData(self.config, self.session)
         self.static = DivaStaticData(self.config, self.session)

titles/diva/schema/__init__.py

@@ -1,6 +1,11 @@
 from titles.diva.schema.profile import DivaProfileData
 from titles.diva.schema.score import DivaScoreData
+from titles.diva.schema.module import DivaModuleData
+from titles.diva.schema.customize import DivaCustomizeItemData
+from titles.diva.schema.pv_customize import DivaPvCustomizeData
 from titles.diva.schema.item import DivaItemData
 from titles.diva.schema.static import DivaStaticData
 
-__all__ = [DivaProfileData, DivaScoreData, DivaItemData, DivaStaticData]
+__all__ = [DivaProfileData, DivaScoreData, DivaModuleData,
+           DivaCustomizeItemData, DivaPvCustomizeData, DivaItemData,
+           DivaStaticData]

titles/diva/schema/customize.py

@@ -0,0 +1,63 @@
+from typing import Optional, Dict, List
+from sqlalchemy import Table, Column, UniqueConstraint, and_
+from sqlalchemy.types import Integer
+from sqlalchemy.schema import ForeignKey
+from sqlalchemy.dialects.mysql import insert
+
+from core.data.schema import BaseData, metadata
+
+customize = Table(
+    "diva_profile_customize_item",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("version", Integer, nullable=False),
+    Column("item_id", Integer, nullable=False),
+    UniqueConstraint("user", "version", "item_id", name="diva_profile_customize_item_uk"),
+    mysql_charset='utf8mb4'
+)
+
+
+class DivaCustomizeItemData(BaseData):
+    def put_customize_item(self, aime_id: int, version: int, item_id: int) -> None:
+        sql = insert(customize).values(
+            version=version,
+            user=aime_id,
+            item_id=item_id
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(f"{__name__} Failed to insert diva profile customize item! aime id: {aime_id} item: {item_id}")
+            return None
+        return result.lastrowid
+
+    def get_customize_items(self, aime_id: int, version: int) -> Optional[List[Dict]]:
+        """
+        Given a game version and an aime id, return all the customize items, not used directly
+        """
+        sql = customize.select(and_(
+                customize.c.version == version,
+                customize.c.user == aime_id
+            ))
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_customize_items_have_string(self, aime_id: int, version: int) -> str:
+        """
+        Given a game version and an aime id, return the cstmz_itm_have hex string
+        required for diva directly
+        """
+        items_list = self.get_customize_items(aime_id, version)
+        if items_list is None:
+            items_list = []
+        item_have = 0
+
+        for item in items_list:
+            item_have |= 1 << item["item_id"]
+
+        # convert the int to a 250 digit long hex string
+        return "{0:0>250}".format(hex(item_have).upper()[2:])