HEPCompSim · HerrHorizontal · Jul 25, 2025 · Jul 25, 2025 · Jul 25, 2025 · Jul 25, 2025
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,7 +1,7 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
 {
-	"name": "Existing Dockerfile",
+	"name": "DCSim devcontainer",
 	"build": {
 		// Sets the run context to one level up instead of the .devcontainer folder.
 		"context": "..",

diff --git a/.gitignore b/.gitignore
@@ -40,6 +40,7 @@ tmp
 *.csv
 *.txt
 tools/tmp*
+tools/**/*.tmp
 
 # logs and tests
 logs
@@ -51,3 +52,4 @@ callgrind.out*
 
 # venv
 .venv/*
+*.DS_Store
diff --git a/Dockerfile b/Dockerfile
@@ -19,6 +19,7 @@ RUN mkdir -p build && pushd build && \
 
 USER root
 RUN pushd build && make install && popd && ldconfig
+
 USER dcsim
 
 # Final image
@@ -31,4 +32,7 @@ COPY --from=builder /home/dcsim/.local /home/dcsim/.local
 COPY --chown=dcsim:dcsim data/ /home/DCSim/data/
 COPY --chown=dcsim:dcsim tools/ /home/DCSim/tools/
 
+# Install tool dependencies
+RUN python3 -m pip install --break-system-packages seaborn
+
 RUN dc-sim --help
diff --git a/data/platform-files/sgbatch_validation_template.xml b/data/platform-files/sgbatch_validation_template.xml
@@ -7,7 +7,7 @@
 
     <zone id="global" routing="Full">
         <zone id="ETP" routing="Floyd">
-            <host id="sg01.etp.kit.edu" speed="{cpu-speed}" core="24">
+            <host id="sg01.etp.kit.edu" speed="{scaled-cpu-speed}" core="24">
                 <prop id="type" value="worker,cache"/>
                 <prop id="ram" value="64TiB"/>
                 <!-- <disk id="ssd_cache1" read_bw="1GBps" write_bw="1GBps"> -->

diff --git a/src/SimpleSimulator.cpp b/src/SimpleSimulator.cpp
@@ -53,8 +53,8 @@ bool SimpleSimulator::infile_caching_on = true;                  // flag to turn
 bool SimpleSimulator::prefetching_on = true;                     // flag to enable prefetching during streaming
 bool SimpleSimulator::shuffle_jobs = false;                      // flag to enable job shuffling during submission
 sg_size_t SimpleSimulator::xrd_block_size = 1000 * 1000 * 1000;  // maximum size of the streamed file blocks in bytes for the XRootD-ish streaming
-double SimpleSimulator::xrd_add_flops_per_time = 20000000000;    // flops overhead introduced by XRootD streaming per second
-double SimpleSimulator::xrd_add_flops_local_per_time = 0;
+double SimpleSimulator::xrd_add_flops_per_time = 1000000;    // flops overhead introduced by XRootD streaming per second
+double SimpleSimulator::xrd_add_flops_local_per_time = 1000000; // flops overhead introduced by streaming from local storage per second
 // TODO: The initialized below is likely bogus (at compile time?)
 std::set<std::string> SimpleSimulator::cache_hosts;
 std::set<std::string> SimpleSimulator::storage_hosts;
@@ -247,8 +247,8 @@ po::variables_map process_program_options(int argc, char **argv) {
     bool shuffle_jobs = false;
 
     sg_size_t xrd_block_size = 1000 * 1000 * 1000;
-    double xrd_add_flops_per_time = 20000000000;
-    double xrd_add_flops_local_per_time = 0;
+    double xrd_add_flops_per_time = 1000000;
+    double xrd_add_flops_local_per_time = 1000000;
     std::string storage_service_buffer_size = "1048576";// 1MiB
 
     unsigned int seed = 42;

diff --git a/tools/__pycache__/simcal_calibrator.cpython-312.pyc b/tools/__pycache__/simcal_calibrator.cpython-312.pyc
diff --git a/tools/evaluate_synthetic.py b/tools/evaluate_synthetic.py
@@ -28,8 +28,8 @@ def main(args):
 						cal[0]["externalSlowNetwork"]=cal[0]["externalNetwork"]
 						del cal[0]["externalNetwork"]
 					for key in cal[0]:
-						if key == "xrootd_flops":
-							continue
+						#if key == "xrootd_flops":
+						#	continue
 						error+=relative_error(args[key],cal[0][key])
 					print(error,cal[1],file)
 					break

diff --git a/tools/example_eval_command.sh b/tools/example_eval_command.sh
@@ -0,0 +1 @@
+./simcal_calibrator.py -a random -t 0 -c 1 -g "$(subRoot.sh)/hep-testjob" -l makespan --keep --include_slow -e "{'cpuSpeed': 1939132803.61, 'cpuSpeed2': 2092187255.84, 'ramDisk':  9206335155.91, 'disk': 33119776.04, 'internalNetwork':  837152846.40, 'xrd_flops_per_time_local':    431422.19, 'xrd_flops_per_time':     325.92, 'externalFastNetwork': 7053334151.98, 'externalSlowNetwork': 187714455.56}"
diff --git a/tools/generate_synthetic.py b/tools/generate_synthetic.py
@@ -117,7 +117,8 @@ def dcsim(self, env, args):
 				 "--cfg=network/loopback-bw:100000000000000",
 				 "--no-caching",
 				 "--seed", 0,
-				 "--xrd-flops-per-time",args["xrootd_flops"]
+				 "--xrd-flops-per-time-local",args["xrd_flops_per_time_local"],
+				 "--xrd-flops-per-time",args["xrd_flops_per_time"],
 			 ]
 		for i in range(len(cargs)):
 			cargs[i]=str(cargs[i])
@@ -134,6 +135,7 @@ def fill_template(self, env, args):
 
 		# Replace the placeholders in the XML file with the specified values
 		xml_contents = re.sub(r'{cpu-speed}', str(args["cpuSpeed"]), xml_contents)
+		xml_contents = re.sub(r'{scaled-cpu-speed}', str(args["cpuSpeed2"]), xml_contents)
 		xml_contents = re.sub(r'{read-speed}', str(args["cacheSpeed"]), xml_contents)
 		xml_contents = re.sub(r'{link-speed}', str(args["internalNetworkSpeed"]), xml_contents)
 		xml_contents = re.sub(r'{net-speed}', str(args["externalNetworkSpeed"]), xml_contents)
@@ -154,7 +156,7 @@ def call_platform(self, env, args):
 			for hitrate in self.hitrates:
 				#print(workload,hitrate,)
 				os.makedirs(args["output"]/workload/args["cacheName"]/args["SGname"], exist_ok=True)
-				i,o=self.dcsim(env,{"workload":self.workloads[workload], "platform":platform.name, "hitrate":hitrate,"xrootd_block":self.xrootd_blocksize,"network_blocksize":self.network_blocksize,"xrootd_flops":args["xrootd_flops"],"output":args["output"]/workload/args["cacheName"]/args["SGname"]/("synthetic_hitrate_"+str(hitrate)+".csv")})
+				i,o=self.dcsim(env,{"workload":self.workloads[workload], "platform":platform.name, "hitrate":hitrate,"xrootd_block":self.xrootd_blocksize,"network_blocksize":self.network_blocksize,"xrd_flops_per_time_local":args["xrd_flops_per_time_local"],"xrd_flops_per_time":args["xrd_flops_per_time"],"output":args["output"]/workload/args["cacheName"]/args["SGname"]/("synthetic_hitrate_"+str(hitrate)+".csv")})
 
 		platform.close()
 
@@ -167,43 +169,51 @@ def run(self, env, args):
 		#scsn = 
 		self.call_platform(env, 
 			{"cpuSpeed": args["cpuSpeed"],
+			"cpuSpeed2": args["cpuSpeed2"],
 			 "cacheSpeed": args["disk"],
 			 "internalNetworkSpeed": args["internalNetwork"],
 			 "externalNetworkSpeed": args["externalSlowNetwork"],
-			 "xrootd_flops":args["xrootd_flops"],
+			 "xrd_flops_per_time":args["xrd_flops_per_time"],
+			 "xrd_flops_per_time_local":args["xrd_flops_per_time_local"],
 			 "output":args["output"],
 			 "cacheName":"diskCache",
 			 "SGname":"SG1_Synthetic1Gbps"
 			 })
 		#fcsn = 
 		self.call_platform(env, 
 			{"cpuSpeed": args["cpuSpeed"],
+			"cpuSpeed2": args["cpuSpeed2"],
 			 "cacheSpeed": args["ramDisk"],
 			 "internalNetworkSpeed": args["internalNetwork"],
 			 "externalNetworkSpeed": args["externalSlowNetwork"],
-			 "xrootd_flops":args["xrootd_flops"],
+			 "xrd_flops_per_time":args["xrd_flops_per_time"],
+			 "xrd_flops_per_time_local":args["xrd_flops_per_time_local"],
 			 "output":args["output"],
 			 "cacheName":"ramCache",
 			 "SGname":"SG1_Synthetic1Gbps"
 			 })
 		#fcfn = 
 		self.call_platform(env, 
 			{"cpuSpeed": args["cpuSpeed"],
+			"cpuSpeed2": args["cpuSpeed2"],
 			 "cacheSpeed": args["ramDisk"],
 			 "internalNetworkSpeed": args["internalNetwork"],
 			 "externalNetworkSpeed": args["externalFastNetwork"],
-			 "xrootd_flops":args["xrootd_flops"],
+			 "xrd_flops_per_time":args["xrd_flops_per_time"],
+			 "xrd_flops_per_time_local":args["xrd_flops_per_time_local"],
 			 "output":args["output"],
 			 "cacheName":"ramCache",
 			 "SGname":"SG1_Synthetic10Gbps"
 			 })
 		#scfn = 
 		self.call_platform(env, 
 			{"cpuSpeed": args["cpuSpeed"],
+			"cpuSpeed2": args["cpuSpeed2"],
 			 "cacheSpeed": args["disk"],
 			 "internalNetworkSpeed": args["internalNetwork"],
 			 "externalNetworkSpeed": args["externalFastNetwork"],
-			 "xrootd_flops":args["xrootd_flops"],
+			 "xrd_flops_per_time":args["xrd_flops_per_time"],
+			 "xrd_flops_per_time_local":args["xrd_flops_per_time_local"],
 			 "output":args["output"],
 			 "cacheName":"diskCache",
 			 "SGname":"SG1_Synthetic10Gbps"
@@ -259,4 +269,7 @@ def run(self, env, args):
 	#print(cal)
 	#print(t1-t0)
 
-#./generate_synthetic.py -g "$(subRoot.sh)/hep-testjob-copy" -o "$(subRoot.sh)/synthetic" -c $(nproc) -a "{'cpuSpeed': 1950000000, 'ramDisk': 27000000000, 'disk': 23000000, 'internalNetwork': 1900000000, 'xrootd_flops': 1000000000000, 'externalFastNetwork': 4000000000, 'externalSlowNetwork': 218000000}"	
+#OUT OF DATE#./generate_synthetic.py -g "$(subRoot.sh)/hep-testjob-copy" -o "$(subRoot.sh)/synthetic" -c $(nproc) -a "{'cpuSpeed': 1950000000, 'ramDisk': 27000000000, 'disk': 23000000, 'internalNetwork': 1900000000, 'xrootd_flops': 1000000000000, 'externalFastNetwork': 4000000000, 'externalSlowNetwork': 218000000}"	
+#above left for record incase we ever need it again
+
+#./generate_synthetic.py -g "$(subRoot.sh)/hep-testjob-copy" -o "$(subRoot.sh)/synthetic" -c $(nproc) -a "{'cpuSpeed': 2197000000, 'cpuSpeed2': 1895000000, 'ramDisk': 2200000000, 'disk': 32000000, 'internalNetwork': 605000000, 'xrd_flops_per_time_local': 19000000, 'xrd_flops_per_time': 10000000, 'externalFastNetwork': 600000000, 'externalSlowNetwork': 73000000}"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		./simcal_calibrator.py -a random -t 0 -c 1 -g "$(subRoot.sh)/hep-testjob" -l makespan --keep --include_slow -e "{'cpuSpeed': 1939132803.61, 'cpuSpeed2': 2092187255.84, 'ramDisk': 9206335155.91, 'disk': 33119776.04, 'internalNetwork': 837152846.40, 'xrd_flops_per_time_local': 431422.19, 'xrd_flops_per_time': 325.92, 'externalFastNetwork': 7053334151.98, 'externalSlowNetwork': 187714455.56}"