5.16.4 improve performance

Author: Michael-X-Net

code/default/launcher/start.py

@@ -120,6 +120,12 @@ def main():
         ports_resolve_solution = win_compat_suggest.Win10PortReserveSolution()
         if not ports_resolve_solution.check_and_resolve():
             return
+    elif sys.platform == "darwin":
+        import resource
+        new_soft_limit = 4096
+        new_hard_limit = 4096
+        resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft_limit, new_hard_limit))
+        xlog.info(f"New open file limits set to: Soft={new_soft_limit}, Hard={new_hard_limit}")
 
     web_control.confirm_xxnet_not_running()
 

code/default/launcher/web_control.py

@@ -1082,7 +1082,7 @@ def start(allow_remote=0):
 
     xlog.info("begin to start web control:%s", addresses)
 
-    server = simple_http_server.HTTPServer(addresses, Http_Handler, logger=xlog)
+    server = simple_http_server.HTTPServer(addresses, Http_Handler, logger=xlog, max_thread=2048)
     server.start()
 
     xlog.info("launcher web control started.")

code/default/launcher/win_compat_suggest.py

@@ -32,6 +32,7 @@ def check_and_resolve(self):
                                   "The service ports was reserved by system, Do you want to change the served port?")
 
             if res == 1:  # Clicked "Yes"
+                xlog.info("User click Yes, start change reserve port range")
                 self.change_reserved_port_range()
 
                 if language == "zh_CN":
@@ -40,6 +41,8 @@ def check_and_resolve(self):
                     self.notify("Computer Restart Required",
                                 "System port reserve range changed, please restart your computer to make chage.")
                 return False
+            else:
+                xlog.info("User click No")
 
         return True
 
@@ -97,6 +100,7 @@ def is_port_reserve_conflict(self):
             # xlog.debug("range:%d - %d", p0, p1)
             for port in self.service_ports:
                 if p0 < port < p1:
+                    xlog.info("found port reserved range:%d - %d, expect %d", p0, p1, port)
                     return True
 
         return False

code/default/lib/noarch/env_info.py

@@ -6,7 +6,6 @@
 
 import utils
 
-
 current_path = os.path.dirname(os.path.abspath(__file__))
 default_path = os.path.abspath(os.path.join(current_path, os.path.pardir, os.path.pardir))
 data_path = os.path.abspath(os.path.join(default_path, os.path.pardir, os.path.pardir, 'data'))
@@ -122,6 +121,59 @@ def get_system_date_path():
     else:
         return os.path.join(home, ".local", "share")
 
+def get_user_data_path():
+    """
+    获取用户数据目录路径
+
+    Returns:
+        Path: 用户数据目录的 Path 对象
+
+    Raises:
+        OSError: 当无法确定用户目录时
+    """
+    try:
+        home = Path.home()
+    except RuntimeError:
+        raise OSError("无法确定用户主目录")
+
+    if sys.platform == "win32":
+        # Windows: %APPDATA%
+        appdata = os.environ.get("APPDATA")
+        if appdata and Path(appdata).exists():
+            return Path(appdata)
+        # Fallback
+        fallback = home / "AppData" / "Roaming"
+        fallback.mkdir(parents=True, exist_ok=True)
+        return fallback
+
+    elif sys.platform == "darwin":
+        # macOS: ~/Library/Application Support
+        app_support = home / "Library" / "Application Support"
+        app_support.mkdir(parents=True, exist_ok=True)
+        return app_support
+
+    else:
+        # Linux/Unix: 遵循 XDG Base Directory Specification
+        xdg_data_home = os.environ.get("XDG_DATA_HOME")
+        if xdg_data_home:
+            data_dir = Path(xdg_data_home)
+        else:
+            data_dir = home / ".local" / "share"
+
+        data_dir.mkdir(parents=True, exist_ok=True)
+        return data_dir
+
+
+def is_in_system_application_path(p):
+    if sys.platform == "win32":
+        if "Program Files" in p:
+            return True
+    elif sys.platform == "darwin":
+        if "Contents" in p:
+            return True
+    else:
+        return False
+
 
 def get_app_name():
     app_info_file = os.path.join(default_path, os.path.pardir, "app_info.json")
@@ -131,16 +183,25 @@ def get_app_name():
         return dat["app_name"]
     except Exception as e:
         print("get app name fail:%r", e)
-    return "XX-Net"
+    return "Dashboard"
 
 
-app_name = get_app_name()
+def use_default_data_path():
+    if os.path.isdir(data_path):
+        return True
 
-# check and update data path
-if not os.path.isdir(data_path):
     try:
         os.mkdir(data_path)
+        return True
     except Exception as e:
-        data_path = os.path.join(get_system_date_path(), app_name)
-        if not os.path.isdir(data_path):
-            os.mkdir(data_path)
+        return False
+
+
+app_name = get_app_name()
+
+# check and update data path
+if not os.path.isdir(data_path) and \
+        (is_in_system_application_path(data_path) or not use_default_data_path() or app_name not in ["XX-Net"]):
+    data_path = os.path.join(get_user_data_path(), app_name)
+    if not os.path.isdir(data_path):
+        os.mkdir(data_path)

code/default/lib/noarch/front_base/boringssl_wrap.py

@@ -20,7 +20,7 @@ def __init__(self, context, sock, ip_str=None, sni=None, on_close=None):
         self._sock = sock
         self._fileno = self._sock.fileno()
         # self._context.logger.debug("sock %s init fd:%d", ip_str, self._fileno)
-        self.ip_str = utils.to_bytes(ip_str)
+        self.ip_str = ip_str
         self.sni = sni
         self._makefile_refs = 0
         self._on_close = on_close

code/default/lib/noarch/front_base/config.py

@@ -14,17 +14,19 @@ def set_default(self):
 
         # http_dispatcher
         self.set_var("dispather_min_idle_workers", 0)
+        self.set_var("dispather_worker_idle_time", 300)
         self.set_var("dispather_work_min_idle_time", 0)
-        self.set_var("dispather_work_max_score", 20000)
+        self.set_var("dispather_work_max_score", 1)
         self.set_var("dispather_min_workers", 0)
         self.set_var("dispather_max_workers", 60)
         self.set_var("dispather_score_factor", 1)
         self.set_var("dispather_max_idle_workers", 30)
         self.set_var("dispather_worker_max_continue_fail", 8)
         self.set_var("dispather_connect_all_workers_on_startup", 0)
-        self.set_var("dispather_ping_check_speed_interval", 300)
+        self.set_var("dispather_ping_check_speed_interval", 60 * 5)
         self.set_var("dispather_ping_upload_size", 1024)
-        self.set_var("dispather_ping_download_size", 10240)
+        self.set_var("dispather_ping_rtt_download_size", 512)
+        self.set_var("dispather_ping_speed_download_size", 1024 * 100)
 
         self.set_var("max_task_num", 100)
 
@@ -48,6 +50,7 @@ def set_default(self):
 
         # worker_base
         self.set_var("show_state_debug", 0)
+        self.set_var("http_query_history_size", 30)  # for calculating rtt and speed.
 
         # connect manager
         self.set_var("https_max_connect_thread", 1)
@@ -70,7 +73,7 @@ def set_default(self):
         self.set_var("check_ip_content", "OK")
 
         # connect_creator
-        self.set_var("socket_timeout", 5)
+        self.set_var("socket_timeout", 1)
         self.set_var("connect_receive_buffer", 1024 * 512)
         self.set_var("connect_send_buffer", 1024 * 512)
         self.set_var("connect_force_http1", 0)
@@ -96,8 +99,13 @@ def set_default(self):
         self.set_var("long_fail_connect_interval", 180)
         self.set_var("short_fail_connect_interval", 10)
         self.set_var("shuffle_ip_on_first_load", 0)
-        self.set_var("ip_speed_history_size", 10)
-        self.set_var("ip_initial_speed", 1000000)
+        self.set_var("ip_speed_history_size", 30)
+        self.set_var("ip_initial_rtt", 0.03)
+        self.set_var("ip_initial_speed", 129000)
+        self.set_var("ip_initial_score", 0.1)
+        self.set_var("ip_cal_rtt_max_package_size", 10000)
+        self.set_var("ip_cal_speed_min_package_size", 100000)
+        self.set_var("ip_cal_expect_time_package_size", 40000)
         self.set_var("ip_speed_save_interval", 60)
 
         # ip source

code/default/lib/noarch/front_base/connect_manager.py

@@ -221,6 +221,8 @@ def keep_connection_daemon(self):
                 time.sleep(5)
                 continue
 
+            if self.config.show_state_debug:
+                self.logger.debug("call _connect_process from keep_connection_daemon")
             self._connect_process()
 
     def _need_more_ip(self):
@@ -230,13 +232,17 @@ def _need_more_ip(self):
             return False
 
     def _create_more_connection(self):
+        if self.config.show_state_debug:
+            self.logger.debug("_create_more_connection")
         if not self.connecting_more_thread:
             with self.thread_num_lock:
                 self.connecting_more_thread = threading.Thread(target=self._create_more_connection_worker,
                                                                name="%s_conn_manager__create_more_conn" % self.logger.name)
                 self.connecting_more_thread.start()
 
     def _create_more_connection_worker(self):
+        if self.config.show_state_debug:
+            self.logger.debug("_create_more_connection_worker")
         if self.start_connect_time and self.start_connect_time + 30 < time.time():
             self.start_connect_time = 0
             self.config.https_max_connect_thread += 1
@@ -260,14 +266,20 @@ def _create_more_connection_worker(self):
             self.connecting_more_thread = None
 
     def _connect_thread(self, sleep_time=0):
+        if self.config.show_state_debug:
+            self.logger.debug("_connect_thread")
+
         if sleep_time > 0.1:
+            self.logger.debug("_connect_thread sleep %f", sleep_time)
             time.sleep(sleep_time)
 
         try:
             while self.running and self._need_more_ip() and time.time() - self.no_ip_time > 10:
                 if self.new_conn_pool.qsize() > self.config.https_connection_pool_max:
                     break
 
+                if self.config.show_state_debug:
+                    self.logger.debug("call _connect_process from _connect_thread")
                 self.start_connect_time = time.time()
                 self._connect_process()
                 self.start_connect_time = 0
@@ -277,19 +289,20 @@ def _connect_thread(self, sleep_time=0):
             self.thread_num_lock.release()
 
     def _connect_process(self):
+        if self.config.show_state_debug:
+            self.logger.debug("_connect_process")
         try:
             host_info = self.ip_manager.get_ip_sni_host()
             if not host_info:
                 self.no_ip_time = time.time()
                 with self.no_ip_lock:
-                    # self.logger.warning("not enough ip")
-                    time.sleep(10)
+                    self.logger.warning("not enough ip")
+                    time.sleep(1)
                 return None
 
             # self.logger.debug("create ssl conn %s", ip_str)
             ssl_sock = self._create_ssl_connection(host_info)
             if not ssl_sock:
-                time.sleep(1)
                 return None
 
             self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
@@ -303,6 +316,9 @@ def _connect_process(self):
             self.logger.exception("connect_process except:%r", e)
 
     def _connect_ssl(self, ip_str, sni, host, close_cb, queue):
+        if self.config.show_state_debug:
+            self.logger.debug("_connect_ssl")
+
         try:
             ssl_sock = self.connect_creator.connect_ssl(ip_str, sni, host, close_cb=close_cb)
             queue.put(ssl_sock)
@@ -311,6 +327,9 @@ def _connect_ssl(self, ip_str, sni, host, close_cb, queue):
             queue.put(e)
 
     def _create_ssl_connection(self, host_info):
+        if self.config.show_state_debug:
+            self.logger.debug("_create_ssl_connection")
+
         ip_str = host_info["ip_str"]
         sni = host_info["sni"]
         host = host_info["host"]
@@ -337,7 +356,7 @@ def _create_ssl_connection(self, host_info):
             return ssl_sock
         except socket.error as e:
             if str(e) in ["no host", "timeout"]:
-                time.sleep(3)
+                pass
             elif not self.check_local_network.is_ok(ip_str):
                 self.logger.debug("connect %s network fail, %r", ip_str, e)
                 time.sleep(1)
@@ -364,15 +383,20 @@ def get_ssl_connection(self, timeout=30):
         end_time = time.time() + timeout
         try:
             while self.running:
-                ret = self.new_conn_pool.get(block=True, timeout=1)
+                ret = self.new_conn_pool.get(block=False)
+                if not ret:
+                    self._create_more_connection()
+                    ret = self.new_conn_pool.get(block=True, timeout=1)
+
                 if ret:
                     handshake_time, ssl_sock = ret
                     if time.time() - ssl_sock.last_use_time < self.config.https_keep_alive - 1:
-                        # self.logger.debug("new_conn_pool.get:%s handshake:%d", ssl_sock.ip, handshake_time)
+                        if self.config.show_state_debug:
+                            self.logger.debug("new_conn_pool.get:%s handshake:%d", ssl_sock.ip, handshake_time)
                         return ssl_sock
                     else:
-                        # self.logger.debug("new_conn_pool.get:%s handshake:%d timeout.", ssl_sock.ip, handshake_time)
-                        self.ip_manager.report_connect_closed(ssl_sock.ip_str, ssl_sock.sni, "get_timeout")
+                        self.logger.warn("new_conn_pool.get:%s handshake:%d timeout.", ssl_sock.ip, handshake_time)
+                        self.ip_manager.report_connect_closed(ssl_sock.ip_str, ssl_sock.sni, "get_ssl_timeout")
                         ssl_sock.close()
                 else:
                     if time.time() > end_time: