Add basic validation scripts

.gitignore

@@ -6,4 +6,5 @@
 /Debug/
 imc_freertos_app_m33.launch
 build/
-build.sh
+build.sh
+imc/exprs/validation/output

CMakeLists.txt

@@ -5,10 +5,11 @@ set(CMAKE_VERBOSE_MAKEFILE 0)
 set(CMAKE_TRY_COMPILE_TARGET_TYPE "STATIC_LIBRARY")
 
 # user configurations: use -D option to change these on command line
-set(IMC_INSERT_COMPILER_CHECKPOINTS 1 CACHE BOOL "" FORCE)
-set(IMC_ENABLE_EXTENSION 1 CACHE BOOL "" FORCE)
-set(IMC_USE_CHECKPOINT_PASS_COUNTER 1 CACHE BOOL "" FORCE)
-set(IMC_USE_CHECKPOINT_VOLTAGE_CHECK 0 CACHE BOOL "" FORCE)
+set(IMC_INSERT_COMPILER_CHECKPOINTS 1 CACHE BOOL "")
+set(IMC_ENABLE_EXTENSION 1 CACHE BOOL "")
+set(IMC_USE_CHECKPOINT_PASS_COUNTER 1 CACHE BOOL "")
+set(IMC_USE_CHECKPOINT_VOLTAGE_CHECK 0 CACHE BOOL "")
+set(IMC_CHECKPOINT_PASS_COUNT 100 CACHE STRING "")
 
 # set(CMAKE_C_COMPILER arm-none-eabi-gcc) #tested with: clang, arm-none-eabi-gcc
 set(CMAKE_C_COMPILER /home/ybkim/workspace/imc/llvm/llvm-17/build/bin/clang)
@@ -123,6 +124,7 @@ if(IMC_USE_CHECKPOINT_PASS_COUNTER)
     endif()
     add_compile_definitions(imcUSE_CHECKPOINT_PASS_COUNTER=1)
     add_compile_definitions(imcUSE_CHECKPOINT_VOLTAGE_CHECK=0)
+    add_compile_definitions(imcCHECKPOINT_PASS_COUNT=${IMC_CHECKPOINT_PASS_COUNT})
 endif()
 
 if(IMC_USE_CHECKPOINT_VOLTAGE_CHECK)
@@ -141,6 +143,7 @@ set(CMAKE_SYSTEM_NAME Generic)
 set(CPU "-mcpu=cortex-m33")
 set(FPU "-mfpu=fpv5-sp-d16")
 set(FLOAT-ABI "-mfloat-abi=hard")
+# set(FLOAT-ABI "-mfloat-abi=soft")
 set(MCU "${CPU} -mthumb ${FPU} ${FLOAT-ABI}")
 set(OPT "-O1")
 set(C_FLAGS_NO_ASM )

Core/Inc/FreeRTOSConfig.h

@@ -59,7 +59,7 @@
 /*-------------------- STM32L5 specific defines -------------------*/
 #define configENABLE_TRUSTZONE                   0
 #define configRUN_FREERTOS_SECURE_ONLY           0
-#define configENABLE_FPU                         0
+#define configENABLE_FPU                         1
 #define configENABLE_MPU                         0
 
 #define configUSE_PREEMPTION                     1

Core/Inc/ImC/imc_extension.h

@@ -9,4 +9,8 @@
     #define imcBENCH_INFINITE_LOOP 0
 #endif
 
+#ifndef imcPRINT_RECOVERY_MESSAGE
+    #define imcPRINT_RECOVERY_MESSAGE 0
+#endif
+
 #endif

Core/Inc/ImC/imc_kernel.h

@@ -18,7 +18,7 @@
 #endif
 
 #ifndef imcCHECKPOINT_VOLTAGE
-    #define imcCHECKPOINT_VOLTAGE 4000 //in mV
+    #define imcCHECKPOINT_VOLTAGE 4100 //in mV
 #endif
 
 #define imcSTM32_CUBE_IDE 0
@@ -40,6 +40,7 @@
 
 void imcInit();
 osThreadId_t imcOsThreadNew(osThreadFunc_t func, void *argument, const osThreadAttr_t *attr);
+void vImcInvalidateRecovery();
 int needCheckpointExecution();
 
 extern int checkpointCounter;

Core/Src/benchmarks/basicmath/basicmath.c

@@ -1,5 +1,6 @@
 #include "benchmarks/basicmath/snipmath.h"
 #include "cmsis_os.h"
+#include "ImC/imc_kernel.h"
 // #include "imc.h"
 
 #include <stdio.h>
@@ -32,6 +33,7 @@ void vBasicMath()
     /* Now solve some random equations */
     for (a1 = 1; a1 < 3; a1++)
     {
+      imcREQUEST_CHECKPOINT();
       for (b1 = 10; b1 > 0; b1--)
       {
         for (c1 = 5; c1 < 15; c1 += 0.5)
@@ -61,7 +63,7 @@ void vBasicMath()
 
   portDISABLE_INTERRUPTS();
   // printf("Sum: %f\r\n", sum);
-  printf("Sum: %d\r\n", (int)sum);
+  printf("(OUT) Sum: %d\r\n", (int)sum);
   portENABLE_INTERRUPTS();
 
   // #if( imcDEBUG_ITM_MESSAGES == 1 )

Core/Src/benchmarks/benchmark_driver.c

@@ -19,11 +19,13 @@ void vBenchmarkDriver(void *_benchmark)
     portDISABLE_INTERRUPTS();
     printf("End benchmark\r\n");
     portENABLE_INTERRUPTS();
+    imcREQUEST_CHECKPOINT();
     #if (imcBENCH_INFINITE_LOOP)
+    osDelay(10);
     }
     #endif
 
-// #if (imcUSE_IMC_KERNEL == 1)
-//     vImcInvalidateRecovery();
-// #endif
+    #if (imcUSE_IMC_KERNEL == 1)
+        vImcInvalidateRecovery();
+    #endif
 }

Core/Src/benchmarks/crc/crc_32.c

@@ -180,7 +180,7 @@ void vCrc()
       register DWORD oldcrc32;
       register int c;
 
-      int IMC_REPEAT = 2000;
+      int IMC_REPEAT = 10;
       // IMC_REPEAT = 6000;
 
       for (int imc_repeat = 0; imc_repeat < IMC_REPEAT; imc_repeat++)
@@ -204,7 +204,7 @@ void vCrc()
             }
       }
 
-      printf("%ld\r\n", oldcrc32);
+      printf("(OUT) %ld\r\n", oldcrc32);
 
       // *crc = oldcrc32 = ~oldcrc32;
 }

Core/Src/benchmarks/fft/fft.c

@@ -207,7 +207,7 @@ void vFFT()
     }
 
     portDISABLE_INTERRUPTS();
-    printf("%d, %d\r\n", (int)(real_sum*1000000), (int)(imag_sum*1000000));
+    printf("(OUT) %d, %d\r\n", (int)(real_sum*1000000), (int)(imag_sum*1000000));
     portENABLE_INTERRUPTS();
 
     // printf("RealOut: ");

Core/Src/benchmarks/sha/sha.c

@@ -233,7 +233,7 @@ void sha_stream(SHA_INFO *sha_info)
 void sha_print(SHA_INFO *sha_info)
 {
     portDISABLE_INTERRUPTS();
-    printf("%08lx %08lx %08lx %08lx %08lx\r\n",
+    printf("(OUT) %08lx %08lx %08lx %08lx %08lx\r\n",
            sha_info->digest[0], sha_info->digest[1], sha_info->digest[2],
            sha_info->digest[3], sha_info->digest[4]);
     portENABLE_INTERRUPTS();
@@ -241,10 +241,11 @@ void sha_print(SHA_INFO *sha_info)
 
 void vSha() {
     SHA_INFO sha_info;
-    int IMC_REPEAT = 50;
+    int IMC_REPEAT = 1;
     // IMC_REPEAT = 70;
     for(int imc_repeat=0; imc_repeat < IMC_REPEAT; imc_repeat++) {
-    sha_stream(&sha_info);
+        memset(&sha_info, 0, sizeof(SHA_INFO));
+        sha_stream(&sha_info);
     } // imc_repeat
     sha_print(&sha_info);
 }

Core/Src/main.c

@@ -149,7 +149,6 @@ void taskEPSRunner(void *argument);
 void taskSnapRunner(void *argument);
 void taskAIRunner(void *argument);
 void taskImcTest(void *argument);
-void taskImcTest_2(void *argument);
 
 /* USER CODE BEGIN PFP */
 /* USER CODE END PFP */
@@ -243,15 +242,16 @@ int main(void)
     imcInit();
   #endif
 
-  printf("\r\n\r\n\r\n");
-  printf("**************************\r\n");
-  printf("** TEST APP INFORMATION **\r\n");
-  printf(" - SBC M33 freeRTOS, truztzone, ADC test\r\n");
-  printf(" - 2023.10.20. 10:00\r\n");
-  printf("**************************\r\n");
-  printf("\r\n\r\n\r\n");
-  /* after SBC boot-up, sbc power gpio pin should be ON */
-  imc_sbc_power_on();	// delay(param) should be 0ms
+  // printf("\r\n\r\n\r\n");
+  // printf("**************************\r\n");
+  // printf("** TEST APP INFORMATION **\r\n");
+  // printf(" - SBC M33 freeRTOS, truztzone, ADC test\r\n");
+  // printf(" - 2023.10.20. 10:00\r\n");
+  // printf("**************************\r\n");
+  // printf("\r\n\r\n\r\n");
+  // /* after SBC boot-up, sbc power gpio pin should be ON */
+  // imc_sbc_power_on();	// delay(param) should be 0ms
+  // printf("+");
 
   /* USER CODE END 2 */
 
@@ -1517,17 +1517,6 @@ void taskImcTest(void *argument) {
   }
 }
 
-void taskImcTest_2(void *argument)
-{
-  int i = 0;
-  while (1)
-  {
-    osDelay(2000);
-    printf("\tj=%d\r\n", i++);
-    // imcREQUEST_CHECKPOINT();
-  }
-}
-
 /**
   * @brief  Period elapsed callback in non blocking mode
   * @note   This function is called  when TIM2 interrupt took place, inside

Middlewares/Third_Party/FreeRTOS/Source/tasks.c

@@ -5308,6 +5308,7 @@ when performing module tests). */
 #endif
 
 #include "ImC/imc_kernel.h"
+#include <stdio.h>
 #if (imcUSE_IMC_KERNEL == 1)
 
 #include "stm32l5xx.h"
@@ -5378,6 +5379,14 @@ void vRecoverTask(TCB_t *tcb)
 	__DMB();
 }
 
+void vImcInvalidateRecovery()
+{
+	for (int i = 0; i < imcMAX_NUM_TASKS; i++)
+	{
+		ucTaskSnapshotValid[i] = pdFALSE;
+	}
+}
+
 osThreadId_t imcOsThreadNew(osThreadFunc_t func, void *argument, const osThreadAttr_t *attr) {
 	osThreadId_t ret = osThreadNew(func, argument, attr);
 	TCB_t *tcb = (TCB_t *)attr->cb_mem;
@@ -5385,6 +5394,9 @@ osThreadId_t imcOsThreadNew(osThreadFunc_t func, void *argument, const osThreadA
 
 	if (ucTaskSnapshotValid[taskNumber] == pdTRUE)
 	{
+#if(imcPRINT_RECOVERY_MESSAGE)
+			printf("Start recovery\r\n");
+#endif
 		vRecoverTask(tcb);
 	}
 	else

imc/exprs/date2025/1_overhead_stable_power/draw_graph_expr_1.ipynb

@@ -0,0 +1,56 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "sys.path.append(\"/home/ybkim/workspace/imc/python\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'imc_utils'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mimc_utils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpps_e36311a\u001b[39;00m\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'imc_utils'"
+     ]
+    }
+   ],
+   "source": [
+    "import imc_utils.pps_e36311a"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "embedded",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

imc/exprs/date2025/1_overhead_stable_power/run_expr_1.py

@@ -0,0 +1,91 @@
+import tempfile
+import time
+
+from imc_utils.pps_e36311a import PPS_E36311A
+from imc_utils.build_config.cortex_m33 import BuildConfigM33
+from imc_utils.build_config.test_env import TestEnv
+import imc_utils.serial_device
+
+# from ..common_configs import WORKSPACE_ROOT, NVM_RESET_BIN, OPENOCD_SCRIPT
+
+WORKSPACE_ROOT = "/home/ybkim/workspace/imc/imc_freertos_app_m33"
+NVM_RESET_BIN = f"{WORKSPACE_ROOT}/imc/utils/nvm_reset.elf"
+OPENOCD_SCRIPT = f"{WORKSPACE_ROOT}/imc_freertos_app_m33.cfg"
+
+def main():
+    pps = PPS_E36311A()
+    config = get_default_build_config()
+
+    config.bench_name = "vBasicMath"
+    config.insert_compiler_checkpoints = True
+    config.use_checkpoint_voltage_check = False
+
+    pps.set_voltage(3.3, 1)
+    pps.set_current(0.1, 1)
+
+    env = TestEnv(WORKSPACE_ROOT, NVM_RESET_BIN, OPENOCD_SCRIPT)
+
+    with tempfile.TemporaryDirectory() as build_dir:
+        binary = env.build_binary(config, build_dir)
+        env.clear_nvm_and_load_binary(binary, resume=True)
+    
+    total_iterations = 5
+    measured_times = measure_execution_time(total_iterations)
+    print(measured_times)
+
+
+def measure_execution_time(total_iterations):
+    ser = imc_utils.serial_device.get_serial()
+
+    num_finished = 0
+    start_detected = False
+    time_takens = []
+    time_total = 0
+
+    ser.read_all()
+
+    while num_finished < total_iterations:
+        if ser.readable():
+            res = ser.readline()
+            try:
+                line = res.decode()[: len(res) - 1]
+            except:
+                continue
+
+        if line.startswith("hardfault"):
+            print("\nHARD FAULT")
+
+        if not start_detected:
+            if line.startswith("Start benchmark"):
+                print("\nbenchmark start detected")
+                t_start = time.time()
+                start_detected = True
+        else:
+            if line.startswith("End benchmark"):
+                t_end = time.time()
+                t_diff = t_end - t_start
+                time_takens.append(t_diff)
+                time_total += t_diff
+                num_finished += 1
+                print(
+                    f"\nbenchmark end detected, time: {t_diff:.2f} secs, finished: {num_finished}, average: {time_total/num_finished:.2f} secs"
+                )
+                start_detected = False
+                # print(time_takens)
+    return time_takens
+
+
+def get_default_build_config():
+    config = BuildConfigM33()
+    config.bench_name = "vBasicMath"
+    config.insert_compiler_checkpoints = True
+    config.enable_extension = True
+    config.use_checkpoint_pass_counter = False
+    config.use_checkpoint_voltage_check = True
+    config.bench_infinite_loop = True
+    return config
+
+
+if __name__ == "__main__":
+    main()
+

imc/exprs/date2025/common_configs.py

@@ -0,0 +1,3 @@
+WORKSPACE_ROOT = "/home/ybkim/workspace/imc/imc_freertos_app_m33"
+NVM_RESET_BIN = f"{WORKSPACE_ROOT}/imc/utils/nvm_reset.elf"
+OPENOCD_SCRIPT = f"{WORKSPACE_ROOT}/imc_freertos_app_m33.cfg"

imc/exprs/validation/validation.py

@@ -0,0 +1,151 @@
+import tempfile
+import time
+import pandas as pd
+import pickle
+
+from imc_utils.pps_e36311a import PPS_E36311A
+from imc_utils.build_config.cortex_m33 import BuildConfigM33
+from imc_utils.build_config.test_env import TestEnv
+import imc_utils.serial_device
+
+WORKSPACE_ROOT = "/home/ybkim/workspace/imc/imc_freertos_app_m33"
+NVM_RESET_BIN = f"{WORKSPACE_ROOT}/imc/utils/nvm_reset.elf"
+OPENOCD_SCRIPT = f"{WORKSPACE_ROOT}/imc_freertos_app_m33.cfg"
+
+def main():
+    pps = PPS_E36311A()
+    config = get_default_build_config()
+
+    benchmarks = [
+        "vBasicMath", "vCrc", "vFFT", "vSha"
+        # "vSha"
+    ]
+
+    for benchmark in benchmarks:
+        config.bench_name = benchmark
+        config.insert_compiler_checkpoints = True
+        config.use_checkpoint_pass_counter = False
+        config.use_checkpoint_voltage_check = True
+        config.checkpoint_pass_count = 100
+        config.bench_infinite_loop = True
+
+        pps.set_voltage(3.3, 1)
+        pps.set_current(0.1, 1)
+        pps.output_on(1)
+
+        env = TestEnv(WORKSPACE_ROOT, NVM_RESET_BIN, OPENOCD_SCRIPT)
+
+        with tempfile.TemporaryDirectory() as build_dir:
+            binary = env.build_binary(config, build_dir)
+            env.clear_nvm_and_load_binary(binary, resume=True)
+        
+        pps.set_current(0.015, 1)
+        time.sleep(1)
+
+        total_iterations = 5
+        records = measure_execution_time(benchmark, total_iterations)
+        df = pd.DataFrame(records)
+        save_records(benchmark, df)
+        print(df)
+
+
+def measure_execution_time(bench_nanme, total_iterations):
+    ser = imc_utils.serial_device.get_serial()
+
+    num_finished = 0
+    start_detected = False
+    time_takens = []
+    time_total = 0
+    outputs = []
+    records = []
+    num_recovery = 0
+
+    ser.reset_input_buffer()
+
+    while num_finished < total_iterations:
+        if ser.readable():
+            res = ser.readline()
+            try:
+                line = res.decode()[: len(res) - 1]
+            except:
+                print("readline() exception")
+                continue
+
+        if line.startswith("hardfault"):
+            print("\nHARD FAULT")
+
+        if not start_detected:
+            if line.startswith("Start benchmark"):
+                print("\nbenchmark start detected")
+                t_start = time.time()
+                start_detected = True
+        else:
+            if line.startswith("End benchmark"):
+                t_end = time.time()
+                t_diff = t_end - t_start
+                time_takens.append(t_diff)
+                time_total += t_diff
+                num_finished += 1
+                print(
+                    f"\nbenchmark end detected, time: {t_diff:.2f} secs, finished: {num_finished}, average: {time_total/num_finished:.2f} secs"
+                )
+                start_detected = False
+                record = {
+                    "bench_name": bench_nanme,
+                    "start": t_start,
+                    "end": t_end,
+                    "time_taken": t_diff,
+                    "recovery": num_recovery,
+                    "outputs": outputs,
+                    "is_correct": check_output_is_correct(bench_nanme, outputs)
+                }
+                records.append(record)
+                outputs = []
+                num_recovery = 0
+
+            elif line.startswith("(OUT)"):
+                print(line)
+                outputs.append(line[6:].strip())
+            
+            elif line.startswith("Start recovery"):
+                num_recovery += 1
+                print(f"recovery: #{num_recovery}", end="\r")
+
+    return records
+
+
+def get_default_build_config():
+    config = BuildConfigM33()
+    config.bench_name = "vBasicMath"
+    config.insert_compiler_checkpoints = True
+    config.enable_extension = True
+    config.use_checkpoint_pass_counter = False
+    config.use_checkpoint_voltage_check = True
+    config.bench_infinite_loop = True
+    config.checkpoint_pass_count = 100
+    config.print_recovery_message = True
+    return config
+
+
+correct_outputs = {
+    "vBasicMath": "Sum: -9313",
+    "vCrc": "210692533",
+    "vFFT": "2807, 915",
+    "vSha": "4926a88d 0ca714f4 a9ebc1eb def37b8e 3911ee0f",
+}
+
+def check_output_is_correct(bench_name, outputs):
+    if len(outputs) == 0 or bench_name not in correct_outputs:
+        return False
+    is_single_output = len(outputs) == 1
+    is_output_correct = correct_outputs[bench_name] == outputs[0]
+    return is_single_output and is_output_correct
+
+
+def save_records(bench_name, df):
+    with open(f"output/{bench_name}.pickle", "wb") as f:
+        pickle.dump(df, f)
+
+
+if __name__ == "__main__":
+    main()

imc_extension.cmake

@@ -1,10 +1,19 @@
-set(IMC_BENCH_NAME "vAes" CACHE STRING "" FORCE)
+set(IMC_BENCH_NAME "vAes" CACHE STRING "")
+set(IMC_BENCH_INFINITE_LOOP 0 CACHE BOOL "")
+set(IMC_PRINT_RECOVERY_MESSAGE 0 CACHE BOOL "")
+
 set(AVAILABLE_BENCHES "vBasicMath" "vStringSearch" "vFFT" "vSha" "vCrc" "vMatMul" "vConv2d" "adc_demo" "vAes")
 if(NOT IMC_BENCH_NAME IN_LIST AVAILABLE_BENCHES)
     message( FATAL_ERROR "incorrect benchmark name: ${IMC_BENCH_NAME}")
 endif()
 
 add_compile_definitions(imcBENCH_NAME=${IMC_BENCH_NAME})
+if(IMC_BENCH_INFINITE_LOOP)
+    add_compile_definitions(imcBENCH_INFINITE_LOOP=1)
+endif()
+if(IMC_PRINT_RECOVERY_MESSAGE)
+    add_compile_definitions(imcPRINT_RECOVERY_MESSAGE=1)
+endif()
 
 if(IMC_BENCH_NAME MATCHES "vConv2d")
     set(BENCHMARK_SRC_FILES

setup_build_dir.py

@@ -0,0 +1,27 @@
+import os
+import shutil
+
+from imc_utils.build_config.cortex_m33 import BuildConfigM33
+from imc_utils.build_config.test_env import TestEnv
+
+WORKSPACE_ROOT = "/home/ybkim/workspace/imc/imc_freertos_app_m33"
+NVM_RESET_BIN = f"{WORKSPACE_ROOT}/imc/utils/nvm_reset.elf"
+OPENOCD_SCRIPT = f"{WORKSPACE_ROOT}/imc_freertos_app_m33.cfg"
+BUILD_DIR = f"{WORKSPACE_ROOT}/build"
+
+config = BuildConfigM33()
+config.bench_name = "vBasicMath"
+config.insert_compiler_checkpoints = False
+config.enable_extension = True
+config.use_checkpoint_pass_counter = False
+config.use_checkpoint_voltage_check = False
+config.bench_infinite_loop = False
+
+env = TestEnv(WORKSPACE_ROOT, NVM_RESET_BIN, OPENOCD_SCRIPT)
+
+if os.path.exists(BUILD_DIR):
+    shutil.rmtree(BUILD_DIR)
+
+os.mkdir(BUILD_DIR)
+
+env.build_binary(config, BUILD_DIR)