From c7137f74a1f9ecf00328fdefb2b34fa79a5df577 Mon Sep 17 00:00:00 2001 From: Martin Bauer Date: Sun, 21 Jun 2020 11:10:45 +0200 Subject: [PATCH] Simpler Measurement session, fully in memory --- firmware/lib/esphttp/WebDAV.cpp | 57 ++++---- firmware/lib/esphttp/WebDAV.h | 8 +- firmware/lib/session/SessionChunk.h | 18 +-- .../lib/session/SimpleMeasurementSession.h | 129 ++++++++++++++++++ firmware/platformio.ini | 4 +- firmware/src/ConfigHardware.h | 16 ++- firmware/src/firmware_main.cpp | 27 ++-- 7 files changed, 206 insertions(+), 53 deletions(-) create mode 100644 firmware/lib/session/SimpleMeasurementSession.h diff --git a/firmware/lib/esphttp/WebDAV.cpp b/firmware/lib/esphttp/WebDAV.cpp index c1282dc..cfeac21 100644 --- a/firmware/lib/esphttp/WebDAV.cpp +++ b/firmware/lib/esphttp/WebDAV.cpp @@ -97,41 +97,44 @@ size_t webdavFileListingSpiffs(char *buffer, size_t maxLength, return bytesWritten; } -String uriToFileName(const String &uriStr) +String uriToFileName(const String &uriStr, const char *spiffsFolder) { - String filename; + String filename; if (uriStr.endsWith(".st")) filename = uriStr.substring(0, uriStr.length() - strlen(".st")); - filename = "/dat/" + filename; + filename = spiffsFolder + String("/") + filename; return filename; } -void webdavHandler(httpd_req_t *req) +std::function webdavHandler(const char *uriPrefix, + const char *spiffsFolder) { - String uri = String(req->uri).substring(strlen("/webdav/")); - constexpr size_t WEBDAV_BUFF_LEN = 1024 * 256; - static char *webdavBuffer = (char *)heap_caps_malloc(WEBDAV_BUFF_LEN, MALLOC_CAP_SPIRAM); - switch (req->method) - { - case HTTP_GET: - case HTTP_PROPFIND: - { - size_t bytesWritten = webdavFileListingSpiffs(webdavBuffer, WEBDAV_BUFF_LEN, "/dat"); - httpd_resp_send(req, webdavBuffer, bytesWritten); - break; - } - case HTTP_DELETE: - { - httpd_resp_set_hdr(req, "Content-Type", "text/plain"); + return [=](httpd_req_t *req) { + String uri = String(req->uri).substring(strlen(uriPrefix)); + constexpr size_t WEBDAV_BUFF_LEN = 1024 * 256; + static char *webdavBuffer = (char *)heap_caps_malloc(WEBDAV_BUFF_LEN, MALLOC_CAP_SPIRAM); + switch (req->method) + { + case HTTP_GET: + case HTTP_PROPFIND: + { + size_t bytesWritten = webdavFileListingSpiffs(webdavBuffer, WEBDAV_BUFF_LEN, spiffsFolder); + httpd_resp_send(req, webdavBuffer, bytesWritten); + break; + } + case HTTP_DELETE: + { + httpd_resp_set_hdr(req, "Content-Type", "text/plain"); - if (portablefs::remove(uriToFileName(uri).c_str())) - httpd_resp_send(req, "Deleted file", -1); - else + if (portablefs::remove(uriToFileName(uri, spiffsFolder).c_str())) + httpd_resp_send(req, "Deleted file", -1); + else + httpd_resp_send_404(req); + break; + } + default: httpd_resp_send_404(req); - break; - } - default: - httpd_resp_send_404(req); - } + } + }; } \ No newline at end of file diff --git a/firmware/lib/esphttp/WebDAV.h b/firmware/lib/esphttp/WebDAV.h index 192ee47..d25582f 100644 --- a/firmware/lib/esphttp/WebDAV.h +++ b/firmware/lib/esphttp/WebDAV.h @@ -1,4 +1,10 @@ #include "Dtypes.h" #include "EspHttp.h" -void webdavHandler(httpd_req_t *req); \ No newline at end of file +/** + * Handler to serves spiffsFolder via webdav + * + * this handler has to be entered for multiple HTTP verbs: HTTP_GET, HTTP_PROPFIND, HTTP_DELETE + */ +std::function webdavHandler(const char *uriPrefix, + const char *spiffsFolder); \ No newline at end of file diff --git a/firmware/lib/session/SessionChunk.h b/firmware/lib/session/SessionChunk.h index a7db596..4c82607 100644 --- a/firmware/lib/session/SessionChunk.h +++ b/firmware/lib/session/SessionChunk.h @@ -1,3 +1,5 @@ +#pragma once + #include "StreamingMsgPackEncoder.h" #include @@ -7,12 +9,12 @@ class SessionChunk { public: SessionChunk() - : nextFree(0), sessionStartTime(0), startIndex(0) + : nextFree_(0), sessionStartTime(0), startIndex(0) {} void init(uint32_t epochStartTime, uint32_t startIdx) { - nextFree = 0; + nextFree_ = 0; sessionStartTime = epochStartTime; startIndex = startIdx; } @@ -26,15 +28,15 @@ public: } uint32_t numMeasurements() const { - return nextFree; + return nextFree_; } bool addPoint(Measurement_T measurement) { - if( nextFree >= SIZE) + if( nextFree_ >= SIZE) return false; - values[nextFree] = measurement; - nextFree++; + values[nextFree_] = measurement; + nextFree_++; return true; } @@ -43,7 +45,7 @@ public: void serialize(StreamingMsgPackEncoder & encoder) const { sendHeader(encoder, sessionStartTime, startIndex); - encoder.sendArray(values, nextFree); + encoder.sendArray(values, nextFree_); } static uint32_t valueOffset() @@ -78,7 +80,7 @@ public: } private: - uint32_t nextFree = 0; + uint32_t nextFree_ = 0; uint32_t sessionStartTime; uint32_t startIndex; Measurement_T values[SIZE]; diff --git a/firmware/lib/session/SimpleMeasurementSession.h b/firmware/lib/session/SimpleMeasurementSession.h new file mode 100644 index 0000000..c3fe569 --- /dev/null +++ b/firmware/lib/session/SimpleMeasurementSession.h @@ -0,0 +1,129 @@ +#include "Dtypes.h" +#include "SessionChunk.h" +#include "FilesystemAbstraction.h" + +template +class SimpleMeasurementSession +{ +public: + using ChunkT = SessionChunk; + + // save interval in number of measurements (by default every minute) + SimpleMeasurementSession(uint32_t saveInterval = 10 * 60) + : chunk(nullptr), saveInterval_(saveInterval) + { + } + ~SimpleMeasurementSession() + { + if (chunk != nullptr) + free(chunk); + } + + void init(uint32_t epochStartTime) + { + if (chunk == nullptr) + { + // psram allocation doesn't seem to work in constructor + chunk = (ChunkT *)heap_caps_malloc(sizeof(ChunkT), MALLOC_CAP_SPIRAM); + new (chunk) ChunkT(); // placement new to init chunk + } + chunk->init(epochStartTime, 0); + } + + bool addPoint(Measurement_T measurement) + { + bool success = chunk->addPoint(measurement); + if (success && (chunk->numMeasurements() % saveInterval_) == 0) + saveToFileSystem(); + if(!success) + Serial.println("Failed to add point"); + //Serial.printf("Add point %d success %d\n", measurement, success); + return success; + } + + void finalize() + { + saveToFileSystem(); + chunk->init(0, 0); + } + + uint32_t getStartTime() const + { + return chunk->getStartTime(); + } + + uint32_t numMeasurements() const + { + return chunk->numMeasurements(); + } + + template + void serialize(Encoder_T &encoder, uint32_t startIdx) const + { + ChunkT::sendHeader(encoder, chunk->getStartTime(), startIdx); + auto numElementsToSend = chunk->numMeasurements() - startIdx; + encoder.sendArray(chunk->getDataPointer() + startIdx, numElementsToSend); + } + +private: + void saveToFileSystem() + { + // todo: check this! free doesn't mean that the file writing actually works ok + // use error codes of write instead? anyway: test it! + deleteUntilBytesFree(CONFIG_SESSION_MAX_SIZE); + + String filename = String(CONFIG_DATA_PATH) + "/" + String(chunk->getStartTime()); + if (portablefs::exists(filename.c_str())) + { + auto file = portablefs::open(filename.c_str(), "a"); + file.seek(0, SeekEnd); + size_t existingMeasurements = (file.size() - ChunkT::valueOffset()) / sizeof(Measurement_T); + Serial.printf("Incremental save, existing %d\n", existingMeasurements); + size_t measurementsToWrite = chunk->numMeasurements() - existingMeasurements; + Measurement_T *startPtr = chunk->getDataPointer() + existingMeasurements; + file.write((uint8_t *)(startPtr), measurementsToWrite * sizeof(Measurement_T)); + } + else + { + Serial.println("First save"); + auto file = portablefs::open(filename.c_str(), "w"); + StreamingMsgPackEncoder encoder(&file); + chunk->serialize(encoder); + } + } + + void deleteUntilBytesFree(size_t requiredSpace) + { + auto freeBytes = portablefs::totalBytes() - portablefs::usedBytes(); + while (freeBytes < requiredSpace) + { + uint32_t nextSessionToDelete = 0; + + auto dir = portablefs::openDir(CONFIG_DATA_PATH); + String filenameToDelete; + while (dir.next()) + { + if (dir.isFile()) + { + const auto fileName = dir.fileName(); + const auto fileNameWithoutDir = fileName.substring(fileName.lastIndexOf("/") + 1); + auto sessionId = fileNameWithoutDir.toInt(); + if (sessionId < nextSessionToDelete) + { + nextSessionToDelete = sessionId; + filenameToDelete = dir.fileName(); + } + } + } + assert(nextSessionToDelete > 0); + Serial.printf("Removing old session %s to make space\n", filenameToDelete.c_str()); + portablefs::remove(filenameToDelete.c_str()); + auto newFreeBytes = portablefs::totalBytes() - portablefs::usedBytes(); + assert(newFreeBytes > freeBytes); + freeBytes = newFreeBytes; + } + } + + ChunkT *chunk; + uint32_t saveInterval_; +}; diff --git a/firmware/platformio.ini b/firmware/platformio.ini index 47c4bf5..4542eaa 100644 --- a/firmware/platformio.ini +++ b/firmware/platformio.ini @@ -13,11 +13,13 @@ data_dir = data [env:esp32] platform = espressif32 +platform_packages = + framework-arduinoespressif32 @ https://github.com/espressif/arduino-esp32.git board = esp-wrover-kit #platform = espressif8266 #board = esp_wroom_02 #build_flags = -Wl,-Teagle.flash.2m1m.ld -build_flags = -DPLATFORM_ESP32 +build_flags = -DPLATFORM_ESP32 -DBOARD_HAS_PSRAM -mfix-esp32-psram-cache-issue framework = arduino monitor_port = /dev/ttyUSB0 upload_port = /dev/ttyUSB0 diff --git a/firmware/src/ConfigHardware.h b/firmware/src/ConfigHardware.h index 389975f..616c537 100644 --- a/firmware/src/ConfigHardware.h +++ b/firmware/src/ConfigHardware.h @@ -2,13 +2,12 @@ #include - //#define _HW_V_20 // HX711 load cell #ifdef PLATFORM_ESP32 -#ifdef _HW_V_20 +#ifdef _HW_V_20 const int CONFIG_SCALE_DOUT_PIN = 23; const int CONFIG_SCALE_SCK_PIN = 22; #else @@ -20,10 +19,15 @@ const int CONFIG_SCALE_SCK_PIN = 23; const int CONFIG_SCALE_DOUT_PIN = D2; const int CONFIG_SCALE_SCK_PIN = D3; #endif -const uint8_t CONFIG_MEASUREMENT_AVG_COUNT = 1; // number of measurements in normal phase -const uint8_t CONFIG_TARE_AVG_COUNT = 6; // number of measurements in tare-phase (to find 0 ) -const int CONFIG_MEASURE_DELAY = 100; // interval in ms between measurements +const uint8_t CONFIG_MEASUREMENT_AVG_COUNT = 1; // number of measurements in normal phase +const uint8_t CONFIG_TARE_AVG_COUNT = 6; // number of measurements in tare-phase (to find 0 ) +const int CONFIG_MEASURE_DELAY = 100; // interval in ms between measurements //const int CONFIG_VALUE_DIVIDER = 8; // uint32 measurements are divided by this factor, before stored in uint16_t -const int CONFIG_VALUE_DIVIDER = 128; // uint32 measurements are divided by this factor, before stored in uint16_t +const int CONFIG_VALUE_DIVIDER = 128; // uint32 measurements are divided by this factor, before stored in uint16_t const uint32_t CONFIG_SESSION_CHUNK_SIZE = 1024; //1024*8 - 16 * sizeof(uint32_t); + +const uint32_t CONFIG_SESSION_MAX_LENGTH_HOURS = 1; +const uint32_t CONFIG_SESSION_MAX_SIZE = CONFIG_SESSION_MAX_LENGTH_HOURS * 3600 * (1000 / CONFIG_MEASURE_DELAY) * sizeof(uint16_t); + +const char *CONFIG_DATA_PATH = "/dat"; \ No newline at end of file diff --git a/firmware/src/firmware_main.cpp b/firmware/src/firmware_main.cpp index 4f1136b..1c4f884 100644 --- a/firmware/src/firmware_main.cpp +++ b/firmware/src/firmware_main.cpp @@ -18,6 +18,7 @@ #include "MeasurementSession.h" #include "SpiffsStorage.h" #include "DeviceInfoLog.h" +#include "SimpleMeasurementSession.h" // Configuration #include "ConfigWifi.h" @@ -29,7 +30,8 @@ WiFiUDP ntpUDP; NTPClient timeClient(ntpUDP, "pool.ntp.org"); -typedef MeasurementSession Session_T; +//typedef MeasurementSession Session_T; +using Session_T = SimpleMeasurementSession; template class SessionManager @@ -46,6 +48,7 @@ public: scale.tare(CONFIG_TARE_AVG_COUNT); Serial.println("Finished tare"); session.init(timeClient.getEpochTime()); + Serial.println("Finished session init"); } void startMeasurements() @@ -69,13 +72,16 @@ public: void iteration() { if (!measuring_) - { - //Serial.println("Disabled"); return; - } + uint16_t measurement = -1; scale.measure(measurement); - session.addPoint(measurement); + bool addPointSuccessful = session.addPoint(measurement); + if(!addPointSuccessful) { + Serial.println("Maximum time of session reached - stopping"); + stopMeasurements(); + return; + } Serial.print("Measurement: "); Serial.println(measurement); if (lastCallTime_ != 0) @@ -198,9 +204,11 @@ void httpSetup(SessionManager *sessionManager) espHttpServer.on("/api/session/data", HTTP_GET, cbGetData); espHttpServer.on("/api/status", HTTP_GET, cbStatus); - espHttpServer.on("/webdav/*?", HTTP_GET, webdavHandler); - espHttpServer.on("/webdav/*?", HTTP_PROPFIND, webdavHandler); - espHttpServer.on("/webdav/*?", HTTP_DELETE, webdavHandler); + auto webdav = webdavHandler("/webdav/", "/dat"); + espHttpServer.on("/webdav/*?", HTTP_GET, webdav); + espHttpServer.on("/webdav/*?", HTTP_PROPFIND, webdav); + espHttpServer.on("/webdav/*?", HTTP_DELETE, webdav); + Serial.println("HTTP setup done"); } void setup() @@ -258,6 +266,5 @@ void setup() void loop() { - delay(10000); - //sessionManager.iteration(); + sessionManager.iteration(); }