Simpler Measurement session, fully in memory

firmware/lib/esphttp/WebDAV.cpp

@@ -97,41 +97,44 @@ size_t webdavFileListingSpiffs(char *buffer, size_t maxLength,
     return bytesWritten;
 }
 
-String uriToFileName(const String &uriStr)
+String uriToFileName(const String &uriStr, const char *spiffsFolder)
 {
-    String filename; 
+    String filename;
     if (uriStr.endsWith(".st"))
         filename = uriStr.substring(0, uriStr.length() - strlen(".st"));
 
-    filename = "/dat/" + filename;
+    filename = spiffsFolder + String("/") + filename;
     return filename;
 }
 
-void webdavHandler(httpd_req_t *req)
+std::function<void(httpd_req_t *)> webdavHandler(const char *uriPrefix,
+                                                 const char *spiffsFolder)
 {
-    String uri = String(req->uri).substring(strlen("/webdav/"));
+    return [=](httpd_req_t *req) {
-    constexpr size_t WEBDAV_BUFF_LEN = 1024 * 256;
+        String uri = String(req->uri).substring(strlen(uriPrefix));
-    static char *webdavBuffer = (char *)heap_caps_malloc(WEBDAV_BUFF_LEN, MALLOC_CAP_SPIRAM);
+        constexpr size_t WEBDAV_BUFF_LEN = 1024 * 256;
-    switch (req->method)
+        static char *webdavBuffer = (char *)heap_caps_malloc(WEBDAV_BUFF_LEN, MALLOC_CAP_SPIRAM);
-    {
+        switch (req->method)
-    case HTTP_GET:
+        {
-    case HTTP_PROPFIND:
+        case HTTP_GET:
-    {
+        case HTTP_PROPFIND:
-        size_t bytesWritten = webdavFileListingSpiffs(webdavBuffer, WEBDAV_BUFF_LEN, "/dat");
+        {
-        httpd_resp_send(req, webdavBuffer, bytesWritten);
+            size_t bytesWritten = webdavFileListingSpiffs(webdavBuffer, WEBDAV_BUFF_LEN, spiffsFolder);
-        break;
+            httpd_resp_send(req, webdavBuffer, bytesWritten);
-    }
+            break;
-    case HTTP_DELETE:
+        }
-    {
+        case HTTP_DELETE:
-        httpd_resp_set_hdr(req, "Content-Type", "text/plain");
+        {
+            httpd_resp_set_hdr(req, "Content-Type", "text/plain");
 
-        if (portablefs::remove(uriToFileName(uri).c_str()))
+            if (portablefs::remove(uriToFileName(uri, spiffsFolder).c_str()))
-            httpd_resp_send(req, "Deleted file", -1);
+                httpd_resp_send(req, "Deleted file", -1);
-        else
+            else
+                httpd_resp_send_404(req);
+            break;
+        }
+        default:
             httpd_resp_send_404(req);
-        break;
+        }
-    }
+    };
-    default:
-        httpd_resp_send_404(req);
-    }
 }

firmware/lib/esphttp/WebDAV.h

@@ -1,4 +1,10 @@
 #include "Dtypes.h"
 #include "EspHttp.h"
 
-void webdavHandler(httpd_req_t *req);
+/**
+ * Handler to serves spiffsFolder via webdav 
+ * 
+ * this handler has to be entered for multiple HTTP verbs: HTTP_GET, HTTP_PROPFIND, HTTP_DELETE
+ */
+std::function<void(httpd_req_t *)> webdavHandler(const char *uriPrefix,
+                                                 const char *spiffsFolder);

firmware/lib/session/SessionChunk.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "StreamingMsgPackEncoder.h"
 #include <cstdint>
 
@@ -7,12 +9,12 @@ class SessionChunk
 {
 public:
     SessionChunk()
-        : nextFree(0), sessionStartTime(0), startIndex(0)
+        : nextFree_(0), sessionStartTime(0), startIndex(0)
     {}
 
     void init(uint32_t epochStartTime, uint32_t startIdx)
     {
-        nextFree = 0;
+        nextFree_ = 0;
         sessionStartTime = epochStartTime;
         startIndex = startIdx;
     }
@@ -26,15 +28,15 @@ public:
     }
 
     uint32_t numMeasurements() const {
-        return nextFree;
+        return nextFree_;
     }
 
     bool addPoint(Measurement_T measurement)
     {
-        if( nextFree >= SIZE)
+        if( nextFree_ >= SIZE)
             return false;
-        values[nextFree] = measurement;
+        values[nextFree_] = measurement;
-        nextFree++;
+        nextFree_++;
 
         return true;
     }
@@ -43,7 +45,7 @@ public:
     void serialize(StreamingMsgPackEncoder<T> & encoder) const
     {
         sendHeader(encoder, sessionStartTime, startIndex);
-        encoder.sendArray(values, nextFree);
+        encoder.sendArray(values, nextFree_);
     }
 
     static uint32_t valueOffset()
@@ -78,7 +80,7 @@ public:
     }
 
 private:
-    uint32_t nextFree = 0;
+    uint32_t nextFree_ = 0;
     uint32_t sessionStartTime;
     uint32_t startIndex;
     Measurement_T values[SIZE];

firmware/lib/session/SimpleMeasurementSession.h

@@ -0,0 +1,129 @@
+#include "Dtypes.h"
+#include "SessionChunk.h"
+#include "FilesystemAbstraction.h"
+
+template <typename Measurement_T, uint32_t MAX_SIZE>
+class SimpleMeasurementSession
+{
+public:
+    using ChunkT = SessionChunk<Measurement_T, MAX_SIZE>;
+
+    // save interval in number of measurements (by default every minute)
+    SimpleMeasurementSession(uint32_t saveInterval = 10 * 60)
+        : chunk(nullptr), saveInterval_(saveInterval)
+    {
+    }
+    ~SimpleMeasurementSession()
+    {
+        if (chunk != nullptr)
+            free(chunk);
+    }
+
+    void init(uint32_t epochStartTime)
+    {
+        if (chunk == nullptr) 
+        {
+            // psram allocation doesn't seem to work in constructor
+            chunk = (ChunkT *)heap_caps_malloc(sizeof(ChunkT), MALLOC_CAP_SPIRAM);
+            new (chunk) ChunkT(); // placement new to init chunk
+        }
+        chunk->init(epochStartTime, 0);
+    }
+
+    bool addPoint(Measurement_T measurement)
+    {
+        bool success = chunk->addPoint(measurement);
+        if (success && (chunk->numMeasurements() % saveInterval_) == 0)
+            saveToFileSystem();
+        if(!success)
+            Serial.println("Failed to add point");
+        //Serial.printf("Add point %d success %d\n", measurement, success);
+        return success;
+    }
+
+    void finalize()
+    {
+        saveToFileSystem();
+        chunk->init(0, 0);
+    }
+
+    uint32_t getStartTime() const
+    {
+        return chunk->getStartTime();
+    }
+
+    uint32_t numMeasurements() const
+    {
+        return chunk->numMeasurements();
+    }
+
+    template <typename Encoder_T>
+    void serialize(Encoder_T &encoder, uint32_t startIdx) const
+    {
+        ChunkT::sendHeader(encoder, chunk->getStartTime(), startIdx);
+        auto numElementsToSend = chunk->numMeasurements() - startIdx;
+        encoder.sendArray(chunk->getDataPointer() + startIdx, numElementsToSend);
+    }
+
+private:
+    void saveToFileSystem()
+    {
+        // todo: check this! free doesn't mean that the file writing actually works ok
+        // use error codes of write instead? anyway: test it!
+        deleteUntilBytesFree(CONFIG_SESSION_MAX_SIZE);
+
+        String filename = String(CONFIG_DATA_PATH) + "/" + String(chunk->getStartTime());
+        if (portablefs::exists(filename.c_str()))
+        {
+            auto file = portablefs::open(filename.c_str(), "a");
+            file.seek(0, SeekEnd);
+            size_t existingMeasurements = (file.size() - ChunkT::valueOffset()) / sizeof(Measurement_T);
+            Serial.printf("Incremental save, existing %d\n", existingMeasurements);
+            size_t measurementsToWrite = chunk->numMeasurements() - existingMeasurements;
+            Measurement_T *startPtr = chunk->getDataPointer() + existingMeasurements;
+            file.write((uint8_t *)(startPtr), measurementsToWrite * sizeof(Measurement_T));
+        }
+        else
+        {
+            Serial.println("First save");
+            auto file = portablefs::open(filename.c_str(), "w");
+            StreamingMsgPackEncoder<portablefs::File> encoder(&file);
+            chunk->serialize(encoder);
+        }
+    }
+
+    void deleteUntilBytesFree(size_t requiredSpace)
+    {
+        auto freeBytes = portablefs::totalBytes() - portablefs::usedBytes();
+        while (freeBytes < requiredSpace)
+        {
+            uint32_t nextSessionToDelete = 0;
+
+            auto dir = portablefs::openDir(CONFIG_DATA_PATH);
+            String filenameToDelete;
+            while (dir.next())
+            {
+                if (dir.isFile())
+                {
+                    const auto fileName = dir.fileName();
+                    const auto fileNameWithoutDir = fileName.substring(fileName.lastIndexOf("/") + 1);
+                    auto sessionId = fileNameWithoutDir.toInt();
+                    if (sessionId < nextSessionToDelete)
+                    {
+                        nextSessionToDelete = sessionId;
+                        filenameToDelete = dir.fileName();
+                    }
+                }
+            }
+            assert(nextSessionToDelete > 0);
+            Serial.printf("Removing old session %s to make space\n", filenameToDelete.c_str());
+            portablefs::remove(filenameToDelete.c_str());
+            auto newFreeBytes = portablefs::totalBytes() - portablefs::usedBytes();
+            assert(newFreeBytes > freeBytes);
+            freeBytes = newFreeBytes;
+        }
+    }
+
+    ChunkT *chunk;
+    uint32_t saveInterval_;
+};

firmware/platformio.ini

@@ -13,11 +13,13 @@ data_dir = data
 
 [env:esp32]
 platform = espressif32
+platform_packages = 
+    framework-arduinoespressif32 @ https://github.com/espressif/arduino-esp32.git
 board = esp-wrover-kit
 #platform = espressif8266
 #board = esp_wroom_02
 #build_flags = -Wl,-Teagle.flash.2m1m.ld
-build_flags = -DPLATFORM_ESP32
+build_flags = -DPLATFORM_ESP32 -DBOARD_HAS_PSRAM -mfix-esp32-psram-cache-issue
 framework = arduino
 monitor_port = /dev/ttyUSB0
 upload_port = /dev/ttyUSB0

firmware/src/ConfigHardware.h

@@ -2,13 +2,12 @@
 
 #include <cstdint>
 
-
 //#define _HW_V_20
 
 // HX711 load cell
 #ifdef PLATFORM_ESP32
 
-#ifdef  _HW_V_20
+#ifdef _HW_V_20
 const int CONFIG_SCALE_DOUT_PIN = 23;
 const int CONFIG_SCALE_SCK_PIN = 22;
 #else
@@ -20,10 +19,15 @@ const int CONFIG_SCALE_SCK_PIN = 23;
 const int CONFIG_SCALE_DOUT_PIN = D2;
 const int CONFIG_SCALE_SCK_PIN = D3;
 #endif
-const uint8_t CONFIG_MEASUREMENT_AVG_COUNT = 1;  // number of measurements in normal phase
+const uint8_t CONFIG_MEASUREMENT_AVG_COUNT = 1; // number of measurements in normal phase
-const uint8_t CONFIG_TARE_AVG_COUNT = 6;         // number of measurements in tare-phase (to find 0 )
+const uint8_t CONFIG_TARE_AVG_COUNT = 6;        // number of measurements in tare-phase (to find 0 )
-const int CONFIG_MEASURE_DELAY = 100;            // interval in ms between measurements
+const int CONFIG_MEASURE_DELAY = 100;           // interval in ms between measurements
 //const int CONFIG_VALUE_DIVIDER = 8;             // uint32 measurements are divided by this factor, before stored in uint16_t
-const int CONFIG_VALUE_DIVIDER = 128;             // uint32 measurements are divided by this factor, before stored in uint16_t
+const int CONFIG_VALUE_DIVIDER = 128; // uint32 measurements are divided by this factor, before stored in uint16_t
 
 const uint32_t CONFIG_SESSION_CHUNK_SIZE = 1024; //1024*8 - 16 * sizeof(uint32_t);
+
+const uint32_t CONFIG_SESSION_MAX_LENGTH_HOURS = 1;
+const uint32_t CONFIG_SESSION_MAX_SIZE = CONFIG_SESSION_MAX_LENGTH_HOURS * 3600 * (1000 / CONFIG_MEASURE_DELAY) * sizeof(uint16_t);
+
+const char *CONFIG_DATA_PATH = "/dat";

firmware/src/firmware_main.cpp

@@ -18,6 +18,7 @@
 #include "MeasurementSession.h"
 #include "SpiffsStorage.h"
 #include "DeviceInfoLog.h"
+#include "SimpleMeasurementSession.h"
 
 // Configuration
 #include "ConfigWifi.h"
@@ -29,7 +30,8 @@
 WiFiUDP ntpUDP;
 NTPClient timeClient(ntpUDP, "pool.ntp.org");
 
-typedef MeasurementSession<uint16_t, SpiffsStorageReader, SpiffsStorageWriter, CONFIG_SESSION_CHUNK_SIZE> Session_T;
+//typedef MeasurementSession<uint16_t, SpiffsStorageReader, SpiffsStorageWriter, CONFIG_SESSION_CHUNK_SIZE> Session_T;
+using Session_T = SimpleMeasurementSession<uint16_t, CONFIG_SESSION_MAX_SIZE>;
 
 template <typename Session_T>
 class SessionManager
@@ -46,6 +48,7 @@ public:
         scale.tare(CONFIG_TARE_AVG_COUNT);
         Serial.println("Finished tare");
         session.init(timeClient.getEpochTime());
+        Serial.println("Finished session init");
     }
 
     void startMeasurements()
@@ -69,13 +72,16 @@ public:
     void iteration()
     {
         if (!measuring_)
-        {
-            //Serial.println("Disabled");
             return;
-        }
+
         uint16_t measurement = -1;
         scale.measure(measurement);
-        session.addPoint(measurement);
+        bool addPointSuccessful = session.addPoint(measurement);
+        if(!addPointSuccessful) {
+            Serial.println("Maximum time of session reached - stopping"); 
+            stopMeasurements();
+            return;
+        }
         Serial.print("Measurement: ");
         Serial.println(measurement);
         if (lastCallTime_ != 0)
@@ -198,9 +204,11 @@ void httpSetup(SessionManager<Session_T> *sessionManager)
     espHttpServer.on("/api/session/data", HTTP_GET, cbGetData);
     espHttpServer.on("/api/status", HTTP_GET, cbStatus);
 
-    espHttpServer.on("/webdav/*?", HTTP_GET, webdavHandler);
+    auto webdav = webdavHandler("/webdav/", "/dat");
-    espHttpServer.on("/webdav/*?", HTTP_PROPFIND, webdavHandler);
+    espHttpServer.on("/webdav/*?", HTTP_GET, webdav);
-    espHttpServer.on("/webdav/*?", HTTP_DELETE, webdavHandler);
+    espHttpServer.on("/webdav/*?", HTTP_PROPFIND, webdav);
+    espHttpServer.on("/webdav/*?", HTTP_DELETE, webdav);
+    Serial.println("HTTP setup done");
 }
 
 void setup()
@@ -258,6 +266,5 @@ void setup()
 
 void loop()
 {
-    delay(10000);
+    sessionManager.iteration();
-    //sessionManager.iteration();
 }