feat: add install and uninstall e2e testing

Author: namchuai

engine/controllers/command_line_parser.cc

@@ -138,15 +138,13 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
     });
 
     auto install_cmd = engines_cmd->add_subcommand("install", "Install engine");
-    install_cmd->callback([] { CLI_LOG("Engine name can't be empty!"); });
     for (auto& engine : engine_service_.kSupportEngines) {
       std::string engine_name{engine};
       EngineInstall(install_cmd, engine_name, version);
     }
 
     auto uninstall_cmd =
         engines_cmd->add_subcommand("uninstall", "Uninstall engine");
-    uninstall_cmd->callback([] { CLI_LOG("Engine name can't be empty!"); });
     for (auto& engine : engine_service_.kSupportEngines) {
       std::string engine_name{engine};
       EngineUninstall(uninstall_cmd, engine_name);

engine/e2e-test/main.py

@@ -1,7 +1,9 @@
 import pytest
 from test_api_engine_list import TestApiEngineList
 from test_cli_engine_get import TestCliEngineGet
+from test_cli_engine_install import TestCliEngineInstall
 from test_cli_engine_list import TestCliEngineList
+from test_cli_engine_uninstall import TestCliEngineUninstall
 
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])

engine/e2e-test/test_cli_engine_get.py

@@ -5,23 +5,52 @@
 
 
 class TestCliEngineGet:
+
+    @pytest.mark.skipif(platform.system() != "Windows", reason="Windows-specific test")
+    def test_engines_get_tensorrt_llm_should_not_be_incompatible(self):
+        exit_code, output, error = run(
+            "Get engine", ["engines", "get", "cortex.tensorrt-llm"]
+        )
+        assert exit_code == 0, f"Get engine failed with error: {error}"
+        assert (
+            "Incompatible" not in output
+        ), "cortex.tensorrt-llm should be Ready or Not Installed on Windows"
+
     @pytest.mark.skipif(platform.system() != "Windows", reason="Windows-specific test")
-    def test_engines_list_run_successfully_on_windows(self):
-        # TODO: implement
-        assert 0 == 0
+    def test_engines_get_onnx_should_not_be_incompatible(self):
+        exit_code, output, error = run("Get engine", ["engines", "get", "cortex.onnx"])
+        assert exit_code == 0, f"Get engine failed with error: {error}"
+        assert (
+            "Incompatible" not in output
+        ), "cortex.onnx should be Ready or Not Installed on Windows"
+
+    def test_engines_get_llamacpp_should_not_be_incompatible(self):
+        exit_code, output, error = run(
+            "Get engine", ["engines", "get", "cortex.llamacpp"]
+        )
+        assert exit_code == 0, f"Get engine failed with error: {error}"
+        assert (
+            "Incompatible" not in output
+        ), "cortex.llamacpp should be compatible for Windows, MacOs and Linux"
 
     @pytest.mark.skipif(platform.system() != "Darwin", reason="macOS-specific test")
-    def test_engines_get_run_successfully_on_macos(self):
-        exit_code, output, error = run("Get engine", ["engines", "get", "cortex.llamacpp"])
+    def test_engines_get_tensorrt_llm_should_be_incompatible_on_macos(self):
+        exit_code, output, error = run(
+            "Get engine", ["engines", "get", "cortex.tensorrt-llm"]
+        )
         assert exit_code == 0, f"Get engine failed with error: {error}"
+        assert (
+            "Incompatible" in output
+        ), "cortex.tensorrt-llm should be Incompatible on MacOS"
 
     @pytest.mark.skipif(platform.system() != "Darwin", reason="macOS-specific test")
-    def test_engines_get_llamacpp_on_macos(self):
-        exit_code, output, error = run("Get engine", ["engines", "get", "cortex.llamacpp"])
+    def test_engines_get_onnx_should_be_incompatible_on_macos(self):
+        exit_code, output, error = run("Get engine", ["engines", "get", "cortex.onnx"])
         assert exit_code == 0, f"Get engine failed with error: {error}"
-        assert "Incompatible" not in output, f"cortex.llamacpp can only be Ready or Not Installed"
+        assert "Incompatible" in output, "cortex.onnx should be Incompatible on MacOS"
 
     @pytest.mark.skipif(platform.system() != "Linux", reason="Linux-specific test")
-    def test_engines_list_run_successfully_on_linux(self):
-        # TODO: implement
-        assert 0 == 0
+    def test_engines_get_onnx_should_be_incompatible_on_linux(self):
+        exit_code, output, error = run("Get engine", ["engines", "get", "cortex.onnx"])
+        assert exit_code == 0, f"Get engine failed with error: {error}"
+        assert "Incompatible" in output, "cortex.onnx should be Incompatible on Linux"

engine/e2e-test/test_cli_engine_install.py

@@ -0,0 +1,30 @@
+import platform
+
+import pytest
+from test_runner import run
+
+
+class TestCliEngineInstall:
+
+    def test_engines_install_llamacpp_should_be_successfully(self):
+        exit_code, output, error = run(
+            "Install Engine", ["engines", "install", "cortex.llamacpp"]
+        )
+        assert "Download" in output, "Should display downloading message"
+        assert exit_code == 0, f"Install engine failed with error: {error}"
+
+    @pytest.mark.skipif(platform.system() != "Darwin", reason="macOS-specific test")
+    def test_engines_install_onnx_on_macos_should_be_failed(self):
+        exit_code, output, error = run(
+            "Install Engine", ["engines", "install", "cortex.onnx"]
+        )
+        assert "No variant found" in output, "Should display error message"
+        assert exit_code == 0, f"Install engine failed with error: {error}"
+
+    @pytest.mark.skipif(platform.system() != "Darwin", reason="macOS-specific test")
+    def test_engines_install_onnx_on_tensorrt_should_be_failed(self):
+        exit_code, output, error = run(
+            "Install Engine", ["engines", "install", "cortex.tensorrt-llm"]
+        )
+        assert "No variant found" in output, "Should display error message"
+        assert exit_code == 0, f"Install engine failed with error: {error}"

engine/e2e-test/test_cli_engine_uninstall.py

@@ -0,0 +1,24 @@
+import pytest
+from test_runner import run
+
+
+class TestCliEngineUninstall:
+
+    @pytest.fixture(autouse=True)
+    def setup_and_teardown(self):
+        # Setup
+        # Preinstall llamacpp engine
+        run("Install Engine", ["engines", "install", "cortex.llamacpp"])
+
+        yield
+
+        # Teardown
+        # Clean up, removing installed engine
+        run("Uninstall Engine", ["engines", "uninsatll", "cortex.llamacpp"])
+
+    def test_engines_uninstall_llamacpp_should_be_successfully(self):
+        exit_code, output, error = run(
+            "Uninstall engine", ["engines", "uninstall", "cortex.llamacpp"]
+        )
+        assert "Engine cortex.llamacpp uninstalled successfully!" in output
+        assert exit_code == 0, f"Install engine failed with error: {error}"

engine/services/engine_service.cc

@@ -71,8 +71,6 @@ std::vector<EngineInfo> EngineService::GetEngineInfoList() const {
 }
 
 void EngineService::UninstallEngine(const std::string& engine) {
-  CTL_INF("Uninstall engine " + engine);
-
   // TODO: Unload the model which is currently running on engine_
 
   // TODO: Unload engine if is loaded