Adding feature to log to SD Card (#708)

* Adding feature to log to SD Card

Co-authored-by: mvgalen <marijnvangalen@gmail.com>

Software/Software.ino

@@ -104,7 +104,7 @@ void setup() {
                           TASK_CONNECTIVITY_PRIO, &connectivity_loop_task, WIFI_CORE);
 #endif
 
-#ifdef LOG_CAN_TO_SD
+#if defined(LOG_CAN_TO_SD) || defined(LOG_TO_SD)
   xTaskCreatePinnedToCore((TaskFunction_t)&logging_loop, "logging_loop", 4096, &logging_task_time_us,
                           TASK_CONNECTIVITY_PRIO, &logging_loop_task, WIFI_CORE);
 #endif
@@ -151,14 +151,19 @@ void loop() {
 #endif
 }
 
-#ifdef LOG_CAN_TO_SD
+#if defined(LOG_CAN_TO_SD) || defined(LOG_TO_SD)
 void logging_loop(void* task_time_us) {
 
-  init_logging_buffer();
+  init_logging_buffers();
   init_sdcard();
 
   while (true) {
+#ifdef LOG_TO_SD
+    write_log_to_sdcard();
+#endif
+#ifdef LOG_CAN_TO_SD
     write_can_frame_to_sdcard();
+#endif
   }
 }
 #endif

Software/USER_SETTINGS.h

@@ -71,9 +71,10 @@
 //#define BMW_SBOX  // SBOX relay control & battery current/voltage measurement
 
 /* Other options */
+//#define LOG_TO_SD              //Enable this line to log diagnostic data to SD card
 //#define DEBUG_VIA_USB          //Enable this line to have the USB port output serial diagnostic data while program runs (WARNING, raises CPU load, do not use for production)
 //#define DEBUG_VIA_WEB          //Enable this line to log diagnostic data while program runs, which can be viewed via webpage (WARNING, slightly raises CPU load, do not use for production)
-#if defined(DEBUG_VIA_USB) || defined(DEBUG_VIA_WEB)
+#if defined(DEBUG_VIA_USB) || defined(DEBUG_VIA_WEB) || defined(LOG_TO_SD)
 #define DEBUG_LOG
 #endif
 

Software/src/devboard/sdcard/sdcard.cpp

@@ -5,11 +5,18 @@
     defined(SD_MISO_PIN)  // ensure code is only compiled if all SD card pins are defined
 
 File can_log_file;
+File log_file;
 RingbufHandle_t can_bufferHandle;
+RingbufHandle_t log_bufferHandle;
 
 bool can_logging_paused = false;
 bool can_file_open = false;
 bool delete_can_file = false;
+
+bool logging_paused = false;
+bool log_file_open = false;
+bool delete_log_file = false;
+
 bool sd_card_active = false;
 
 void delete_can_log() {
@@ -27,6 +34,26 @@ void pause_can_writing() {
   can_logging_paused = true;
 }
 
+void delete_log() {
+  logging_paused = true;
+  if (log_file_open) {
+    log_file.close();
+    log_file_open = false;
+  }
+  SD.remove(LOG_FILE);
+  logging_paused = false;
+}
+
+void resume_log_writing() {
+  logging_paused = false;
+  log_file = SD.open(LOG_FILE, FILE_APPEND);
+  log_file_open = true;
+}
+
+void pause_log_writing() {
+  logging_paused = true;
+}
+
 void add_can_frame_to_buffer(CAN_frame frame, frameDirection msgDir) {
 
   if (!sd_card_active)
@@ -84,17 +111,65 @@ void write_can_frame_to_sdcard() {
         can_log_file.print(" ");
     }
     can_log_file.println("");
+    can_log_file.flush();
 
     vRingbufferReturnItem(can_bufferHandle, (void*)log_frame);
   }
 }
 
-void init_logging_buffer() {
-  can_bufferHandle = xRingbufferCreate(64 * 1024, RINGBUF_TYPE_BYTEBUF);
+void add_log_to_buffer(const uint8_t* buffer, size_t size) {
+
+  if (!sd_card_active)
+    return;
+
+  if (xRingbufferSend(log_bufferHandle, buffer, size, 0) != pdTRUE) {
+    Serial.println("Failed to send log to ring buffer!");
+    return;
+  }
+}
+
+void write_log_to_sdcard() {
+
+  if (!sd_card_active)
+    return;
+
+  size_t receivedMessageSize;
+  uint8_t* buffer = (uint8_t*)xRingbufferReceive(log_bufferHandle, &receivedMessageSize, pdMS_TO_TICKS(10));
+
+  if (buffer != NULL) {
+
+    if (logging_paused) {
+      vRingbufferReturnItem(log_bufferHandle, (void*)buffer);
+      return;
+    }
+
+    if (log_file_open == false) {
+      log_file = SD.open(LOG_FILE, FILE_APPEND);
+      log_file_open = true;
+    }
+
+    log_file.write(buffer, receivedMessageSize);
+    log_file.flush();
+    vRingbufferReturnItem(log_bufferHandle, (void*)buffer);
+  }
+}
+
+void init_logging_buffers() {
+#if defined(LOG_CAN_TO_SD)
+  can_bufferHandle = xRingbufferCreate(32 * 1024, RINGBUF_TYPE_BYTEBUF);
   if (can_bufferHandle == NULL) {
     Serial.println("Failed to create CAN ring buffer!");
     return;
   }
+#endif  // defined(LOG_CAN_TO_SD)
+
+#if defined(LOG_TO_SD)
+  log_bufferHandle = xRingbufferCreate(1024, RINGBUF_TYPE_BYTEBUF);
+  if (log_bufferHandle == NULL) {
+    Serial.println("Failed to create log ring buffer!");
+    return;
+  }
+#endif  // defined(LOG_TO_SD)
 }
 
 void init_sdcard() {

Software/src/devboard/sdcard/sdcard.h

@@ -9,8 +9,9 @@
 #if defined(SD_CS_PIN) && defined(SD_SCLK_PIN) && defined(SD_MOSI_PIN) && \
     defined(SD_MISO_PIN)  // ensure code is only compiled if all SD card pins are defined
 #define CAN_LOG_FILE "/canlog.txt"
+#define LOG_FILE "/log.txt"
 
-void init_logging_buffer();
+void init_logging_buffers();
 
 void init_sdcard();
 void print_sdcard_details();
@@ -21,6 +22,12 @@ void write_can_frame_to_sdcard();
 void pause_can_writing();
 void resume_can_writing();
 void delete_can_log();
-#endif  // defined(SD_CS_PIN) && defined(SD_SCLK_PIN) && defined(SD_MOSI_PIN) && defined(SD_MISO_PIN)
+void delete_log();
+void resume_log_writing();
+void pause_log_writing();
 
+void add_log_to_buffer(const uint8_t* buffer, size_t size);
+void write_log_to_sdcard();
+
+#endif  // defined(SD_CS_PIN) && defined(SD_SCLK_PIN) && defined(SD_MOSI_PIN) && defined(SD_MISO_PIN)
 #endif  // SDCARD_H

Software/src/devboard/utils/logging.cpp

@@ -1,38 +1,84 @@
 #include "logging.h"
 #include "../../datalayer/datalayer.h"
+#include "../sdcard/sdcard.h"
 
-size_t Logging::write(const uint8_t* buffer, size_t size) {
+#define MAX_LINE_LENGTH_PRINTF 128
+#define MAX_LENGTH_TIME_STR 14
+
+bool previous_message_was_newline = true;
+
+void Logging::add_timestamp(size_t size) {
 #ifdef DEBUG_LOG
   char* message_string = datalayer.system.info.logged_can_messages;
   int offset = datalayer.system.info.logged_can_messages_offset;  // Keeps track of the current position in the buffer
   size_t message_string_size = sizeof(datalayer.system.info.logged_can_messages);
   unsigned long currentTime = millis();
-#ifdef DEBUG_VIA_USB
-  size_t n = 0;
-  while (size--) {
-    if (Serial.write(*buffer++))
-      n++;
-    else
-      break;
+  char* timestr;
+  static char timestr_buffer[MAX_LENGTH_TIME_STR];
+
+#ifdef DEBUG_VIA_WEB
+  if (!datalayer.system.info.can_logging_active) {
+    /* If web debug is active and can logging is inactive, 
+     * we use the debug logging memory directly for writing the timestring */
+    if (offset + size + MAX_LENGTH_TIME_STR > message_string_size) {
+      offset = 0;
+    }
+    timestr = datalayer.system.info.logged_can_messages + offset;
+  } else {
+    timestr = timestr_buffer;
   }
-  return n;
+#else
+  timestr = timestr_buffer;
+#endif  // DEBUG_VIA_WEB
+
+  offset += min(MAX_LENGTH_TIME_STR - 1,
+                snprintf(timestr, MAX_LENGTH_TIME_STR, "%8lu.%03lu ", currentTime / 1000, currentTime % 1000));
+
+#ifdef DEBUG_VIA_WEB
+  if (!datalayer.system.info.can_logging_active) {
+    datalayer.system.info.logged_can_messages_offset = offset;  // Update offset in buffer
+  }
+#endif  // DEBUG_VIA_WEB
+
+#ifdef LOG_TO_SD
+  add_log_to_buffer((uint8_t*)timestr, MAX_LENGTH_TIME_STR);
+#endif  // LOG_TO_SD
+
+#ifdef DEBUG_VIA_USB
+  Serial.write(timestr);
+#endif  // DEBUG_VIA_USB
+
+#endif  // DEBUG_LOG
+}
+
+size_t Logging::write(const uint8_t* buffer, size_t size) {
+#ifdef DEBUG_LOG
+  if (previous_message_was_newline) {
+    add_timestamp(size);
+  }
+
+#ifdef LOG_TO_SD
+  add_log_to_buffer(buffer, size);
+#endif
+#ifdef DEBUG_VIA_USB
+  Serial.write(buffer, size);
 #endif
 #ifdef DEBUG_VIA_WEB
-  if (datalayer.system.info.can_logging_active) {
-    return 0;
+  if (!datalayer.system.info.can_logging_active) {
+    char* message_string = datalayer.system.info.logged_can_messages;
+    int offset = datalayer.system.info.logged_can_messages_offset;  // Keeps track of the current position in the buffer
+    size_t message_string_size = sizeof(datalayer.system.info.logged_can_messages);
+
+    if (offset + size > message_string_size) {
+      offset = 0;
+    }
+    memcpy(message_string + offset, buffer, size);
+    datalayer.system.info.logged_can_messages_offset = offset + size;  // Update offset in buffer
   }
-  if (offset + size + 13 > sizeof(datalayer.system.info.logged_can_messages)) {
-    offset = 0;
-  }
-  if (buffer[0] != '\r' && buffer[0] != '\n' &&
-      (offset == 0 || message_string[offset - 1] == '\r' || message_string[offset - 1] == '\n')) {
-    offset += snprintf(message_string + offset, message_string_size - offset - 1, "%8lu.%03lu ", currentTime / 1000,
-                       currentTime % 1000);
-  }
-  memcpy(message_string + offset, buffer, size);
-  datalayer.system.info.logged_can_messages_offset = offset + size;  // Update offset in buffer
-  return size;
 #endif  // DEBUG_VIA_WEB
+
+  previous_message_was_newline = buffer[size - 1] == '\n';
+  return size;
 #endif  // DEBUG_LOG
   return 0;
 }
@@ -42,45 +88,50 @@ void Logging::printf(const char* fmt, ...) {
   char* message_string = datalayer.system.info.logged_can_messages;
   int offset = datalayer.system.info.logged_can_messages_offset;  // Keeps track of the current position in the buffer
   size_t message_string_size = sizeof(datalayer.system.info.logged_can_messages);
-#ifdef DEBUG_VIA_USB
-  static char buf[128];
-  message_string = buf;
-  offset = 0;
-  message_string_size = sizeof(buf);
-#endif
+
+  if (previous_message_was_newline) {
+    add_timestamp(MAX_LINE_LENGTH_PRINTF);
+  }
+
+  static char buffer[MAX_LINE_LENGTH_PRINTF];
+  char* message_buffer;
 #ifdef DEBUG_VIA_WEB
-  if (datalayer.system.info.can_logging_active) {
-    return;
+  if (!datalayer.system.info.can_logging_active) {
+    /* If web debug is active and can logging is inactive, 
+     * we use the debug logging memory directly for writing the output */
+    if (offset + MAX_LINE_LENGTH_PRINTF > message_string_size) {
+      // Not enough space, reset and start from the beginning
+      offset = 0;
+    }
+    message_buffer = message_string + offset;
+  } else {
+    message_buffer = buffer;
   }
-  message_string = datalayer.system.info.logged_can_messages;
-  offset = datalayer.system.info.logged_can_messages_offset;  // Keeps track of the current position in the buffer
-  message_string_size = sizeof(datalayer.system.info.logged_can_messages);
-#endif
-  if (offset + 128 > sizeof(datalayer.system.info.logged_can_messages)) {
-    // Not enough space, reset and start from the beginning
-    offset = 0;
-  }
-  unsigned long currentTime = millis();
-  // Add timestamp
-  offset += snprintf(message_string + offset, message_string_size - offset - 1, "%8lu.%03lu ", currentTime / 1000,
-                     currentTime % 1000);
+#else
+  message_buffer = buffer;
+#endif  // DEBUG_VIA_WEB
 
   va_list(args);
   va_start(args, fmt);
-  offset += vsnprintf(message_string + offset, message_string_size - offset - 1, fmt, args);
+  int size = min(MAX_LINE_LENGTH_PRINTF - 1, vsnprintf(message_buffer, MAX_LINE_LENGTH_PRINTF, fmt, args));
   va_end(args);
 
-  if (datalayer.system.info.can_logging_active) {
-    size_t size = offset;
-    size_t n = 0;
-    while (size--) {
-      if (Serial.write(*message_string++))
-        n++;
-      else
-        break;
-    }
-  } else {
-    datalayer.system.info.logged_can_messages_offset = offset;  // Update offset in buffer
+#ifdef LOG_TO_SD
+  add_log_to_buffer((uint8_t*)message_buffer, size);
+#endif  // LOG_TO_SD
+
+#ifdef DEBUG_VIA_USB
+  Serial.write(message_buffer, size);
+#endif  // DEBUG_VIA_USB
+
+#ifdef DEBUG_VIA_WEB
+  if (!datalayer.system.info.can_logging_active) {
+    // Data was already added to buffer, just move offset
+    datalayer.system.info.logged_can_messages_offset =
+        offset + size;  // Keeps track of the current position in the buffer
   }
+#endif  // DEBUG_VIA_WEB
+
+  previous_message_was_newline = buffer[size - 1] == '\n';
 #endif  // DEBUG_LOG
 }

Software/src/devboard/utils/logging.h

@@ -5,6 +5,8 @@
 #include "Print.h"
 
 class Logging : public Print {
+  void add_timestamp(size_t size);
+
  public:
   virtual size_t write(const uint8_t* buffer, size_t size);
   virtual size_t write(uint8_t) { return 0; }

Software/src/devboard/webserver/debug_logging_html.cpp

@@ -3,7 +3,7 @@
 #include "../../datalayer/datalayer.h"
 #include "index_html.h"
 
-#ifdef DEBUG_VIA_WEB
+#if defined(DEBUG_VIA_WEB) || defined(LOG_TO_SD)
 String debug_logger_processor(void) {
   String content = String(index_html_header);
   // Page format
@@ -17,8 +17,13 @@ String debug_logger_processor(void) {
       ".can-message { background-color: #404E57; margin-bottom: 5px; padding: 10px; border-radius: 5px; font-family: "
       "monospace; }";
   content += "</style>";
+#ifdef DEBUG_VIA_WEB
   content += "<button onclick='refreshPage()'>Refresh data</button> ";
+#endif
   content += "<button onclick='exportLog()'>Export to .txt</button> ";
+#ifdef LOG_TO_SD
+  content += "<button onclick='deleteLog()'>Delete log file</button> ";
+#endif
   content += "<button onclick='goToMainPage()'>Back to main page</button>";
 
   // Start a new block for the debug log messages
@@ -30,9 +35,12 @@ String debug_logger_processor(void) {
   content += "<script>";
   content += "function refreshPage(){ location.reload(true); }";
   content += "function exportLog() { window.location.href = '/export_log'; }";
+#ifdef LOG_TO_SD
+  content += "function deleteLog() { window.location.href = '/delete_log'; }";
+#endif
   content += "function goToMainPage() { window.location.href = '/'; }";
   content += "</script>";
   content += index_html_footer;
   return content;
 }
-#endif  // DEBUG_VIA_WEB
+#endif

Software/src/devboard/webserver/webserver.cpp

@@ -65,7 +65,7 @@ void init_webserver() {
     request->send(response);
   });
 
-#ifdef DEBUG_VIA_WEB
+#if defined(DEBUG_VIA_WEB) || defined(LOG_TO_SD)
   // Route for going to debug logging web page
   server.on("/log", HTTP_GET, [](AsyncWebServerRequest* request) {
     AsyncWebServerResponse* response = request->beginResponse(200, "text/html", debug_logger_processor());
@@ -123,6 +123,22 @@ void init_webserver() {
   });
 #endif
 
+#ifdef LOG_TO_SD
+  // Define the handler to delete log file
+  server.on("/delete_log", HTTP_GET, [](AsyncWebServerRequest* request) {
+    delete_log();
+    request->send_P(200, "text/plain", "Log file deleted");
+  });
+
+  // Define the handler to export debug log
+  server.on("/export_log", HTTP_GET, [](AsyncWebServerRequest* request) {
+    pause_log_writing();
+    request->send(SD, LOG_FILE, String(), true);
+    resume_log_writing();
+  });
+#endif
+
+#ifndef LOG_TO_SD
   // Define the handler to export debug log
   server.on("/export_log", HTTP_GET, [](AsyncWebServerRequest* request) {
     String logs = String(datalayer.system.info.logged_can_messages);
@@ -149,6 +165,7 @@ void init_webserver() {
     response->addHeader("Content-Disposition", String("attachment; filename=\"") + String(filename) + "\"");
     request->send(response);
   });
+#endif
 
   // Route for going to cellmonitor web page
   server.on("/cellmonitor", HTTP_GET, [](AsyncWebServerRequest* request) {
@@ -1061,7 +1078,7 @@ String processor(const String& var) {
     content += "<button onclick='Settings()'>Change Settings</button> ";
     content += "<button onclick='Advanced()'>More Battery Info</button> ";
     content += "<button onclick='CANlog()'>CAN logger</button> ";
-#ifdef DEBUG_VIA_WEB
+#if defined(DEBUG_VIA_WEB) || defined(LOG_TO_SD)
     content += "<button onclick='Log()'>Log</button> ";
 #endif  // DEBUG_VIA_WEB
     content += "<button onclick='Cellmon()'>Cellmonitor</button> ";

Software/src/devboard/wifi/wifi.cpp

@@ -206,7 +206,7 @@ void init_WiFi_AP() {
 #ifdef DEBUG_LOG
   logging.println("Access Point created.");
   logging.print("IP address: ");
-  logging.println(IP);
+  logging.println(IP.toString());
 #endif
 }
 #endif  // WIFIAP

Software/src/include.h

@@ -48,7 +48,7 @@
 #error No battery selected! Choose one from the USER_SETTINGS.h file
 #endif
 
-#ifdef LOG_CAN_TO_SD
+#if defined(LOG_CAN_TO_SD) || defined(LOG_TO_SD)
 #if !defined(HW_LILYGO)
 #error The SD card logging feature is only available on LilyGo hardware
 #endif