Fixing deploy script

firmware/.gitignore

@@ -3,3 +3,5 @@
 .vscode/c_cpp_properties.json
 .vscode/launch.json
 .vscode/ipch
+/.cache
+compile_commands.json

firmware/.vscode/extensions.json

@@ -3,5 +3,8 @@
     // for the documentation about the extensions.json format
     "recommendations": [
         "platformio.platformio-ide"
+    ],
+    "unwantedRecommendations": [
+        "ms-vscode.cpptools-extension-pack"
     ]
 }

firmware/deploy-firmware.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+
+import struct
+from collections import namedtuple
+from datetime import datetime
+import subprocess
+import packaging.version
+from esptool.bin_image import LoadFirmwareImage  # required to get hash and checksum right
+import os
+
+# locale.setlocale(locale.LC_ALL, 'en_US')
+
+APP_DESC_SIZE = 256 # sizeof(esp_app_desc_t)
+
+APP_DESC_STRUCT = "<II2I32s32s16s16s32s32B20I"
+AppDesc = namedtuple('AppDesc',
+                     [
+                         "secure_version",
+                         "version",
+                         "project_name",
+                         "time",
+                         "date",
+                         "idf_ver",
+                         "app_elf_sha256",
+                     ]
+                     )
+
+
+def version_number_from_git(tag_prefix='release/', sha_length=10, version_format="{version}.dev{commits}+{sha}"):
+    def get_released_versions():
+        tags = sorted(subprocess.getoutput('git tag').split('\n'))
+        versions = [t[len(tag_prefix):]
+                    for t in tags if t.startswith(tag_prefix)]
+        return versions
+
+    def tag_from_version(v):
+        return tag_prefix + str(v)
+
+    def increment_version(v: packaging.version.Version):
+        parsed_version = [int(i) for i in str(v).split('.')]
+        parsed_version[-1] += 1
+        return packaging.version.parse('.'.join(str(i) for i in parsed_version))
+
+    versions = [packaging.version.parse(s) for s in get_released_versions()]
+    versions.sort()
+    latest_release = versions[-1]
+    commits_since_tag = subprocess.getoutput(
+        'git rev-list {}..HEAD --count'.format(tag_from_version(latest_release)))
+    sha = subprocess.getoutput('git rev-parse HEAD')[:sha_length]
+    is_dirty = len(subprocess.getoutput(
+        "git status --untracked-files=no -s")) > 0
+
+    if int(commits_since_tag) == 0:
+        version_string = str(latest_release)
+    else:
+        next_version = increment_version(latest_release)
+        version_string = version_format.format(version=next_version, commits=commits_since_tag, sha=sha)
+
+    if is_dirty:
+        version_string += ".dirty"
+    return version_string
+
+
+def read_app_description_from_segment(segment):
+    def process_bytes(b):
+        if not isinstance(b, bytes):
+            return b
+        s = b.decode()
+        return s[:s.find("\x00")]
+
+    unpacked = struct.unpack(APP_DESC_STRUCT, segment.data[:APP_DESC_SIZE])
+    unpacked = tuple(process_bytes(e) for e in unpacked)
+    magic_word, secure_version, _, _, version, project_name, time, date, idf_ver, app_elf_sha256, *_ = unpacked
+    assert magic_word == 0xABCD5432
+    return AppDesc(secure_version, version, project_name, time, date, idf_ver, app_elf_sha256)
+
+
+
+def patch_app_description_in_segment(segment, version, project_name, time, date):
+    #assert len(version) < 32
+    assert len(project_name) < 32
+    assert len(time) < 16
+    assert len(date) < 16
+
+    def fill_zeros(s, total_length):
+        s += "\x00" * (total_length - len(s))
+        s = s.encode()
+        assert len(s) == total_length
+        return s
+
+    raw_app_desc = segment.data[:APP_DESC_SIZE]
+    unpacked = list(struct.unpack(APP_DESC_STRUCT, raw_app_desc))
+    unpacked[4] = fill_zeros(version, 32)
+    unpacked[5] = fill_zeros(project_name, 32)
+    unpacked[6] = fill_zeros(time, 16)
+    unpacked[7] = fill_zeros(date, 16)
+    packed = struct.pack(APP_DESC_STRUCT, *unpacked)
+
+    original_data = segment.data
+    segment.data = packed + original_data[APP_DESC_SIZE:]
+
+
+def patch_firmware(input_file, output_file):
+    img = LoadFirmwareImage("esp32", input_file)
+    version = version_number_from_git()
+
+    now = datetime.now()
+    time = now.strftime("%H:%M:%S")
+    date = now.strftime("%b %d %Y")
+    patch_app_description_in_segment(img.segments[0], version, "swimtracker.bauer.tech", time, date)
+    img.save(output_file)
+
+
+if __name__ == "__main__":
+    firmware_file = ".pio/build/esp32/firmware.bin"
+    version_file_name = "VERSION"
+
+    patch_firmware(firmware_file, firmware_file) # patch inplace
+
+    version = version_number_from_git()
+    print("Deploying ", version)
+    with open(version_file_name, "w") as version_file:
+      print(version, file=version_file)
+
+    subprocess.run(["scp", firmware_file, "core@server:/docker/web/volumes/static-sites/swimtracker-update"])
+    subprocess.run(["scp", version_file_name, "core@server:/docker/web/volumes/static-sites/swimtracker-update"])
+
+    os.unlink(firmware_file)
+    os.unlink(version_file_name)
+

firmware/dev_hardware_wires.md

@@ -0,0 +1,15 @@
+!! On Swimtracker board the JTAG pins are not labeled correctly ! 
+one pin even needs to be soldered manually
+
+
+------|------|-----------------
+GND   |      | schwarz
+3.3V  |      | rot
+IO13  | TCK  | weiss
+IO14  | TMS  | grau
+IO12  | TDI  | lila
+IO15  | TDO  | blau
+EN    |      | grun
+IO0   |      | braun
+RXD   |      | orange
+TXD   |      | gelb

firmware/esp-idf-install.readme

@@ -0,0 +1,20 @@
+git clone -b v3.3 --recursive https://github.com/espressif/esp-idf.git
+set -x IDF_TOOLS_PATH /home/martin/code/swimtracker-firmware/idf-tools
+cd esp-idf
+./install.fish
+
+# install arduino
+mdir components
+cd components
+git clone https://github.com/espressif/arduino-esp32.git arduino
+
+# to get into environment (works only bash?)
+export IDF_TOOLS_PATH=/home/martin/code/swimtracker-firmware/idf-tools/
+.  /home/martin/code/swimtracker-firmware/esp-idf/export.sh
+
+
++ use only arduino libraries that are really necessary => smaller firmware 
+
+-> result: new esp-idf version doesn't work together with arduino-esp
+-> use 3.3? 
+-> not yet clear: how to use arduino component libraries

firmware/lib/basic/AllocAbstraction.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#ifdef PLATFORM_NATIVE
+
+#include <stdlib.h>
+
+static constexpr uint32_t MALLOC_CAP_SPIRAM = -1;
+
+inline void *heap_caps_malloc(size_t size, uint32_t /*caps*/) {
+  return std::malloc(size);
+}
+
+#endif

firmware/lib/basic/FilesystemAbstraction.h

@@ -8,6 +8,12 @@
 namespace portablefs
 {
     using File = ::File;
+    using string = String;
+
+    template<typename T> 
+    String to_string(const T & value) {
+        return String(value);
+    }
 
     class Dir
     {
@@ -85,20 +91,11 @@ namespace portablefs
         return SPIFFS.totalBytes();
     }
 
+
 } // namespace portablefs
 
 #endif
 
-#ifdef PLATFORM_ESP8266
-
-#include <FS.h>
-
-namespace portablefs
-{
-    using Dir;
-} // namespace portablefs
-
-#endif
 
 #ifdef PLATFORM_NATIVE
 
@@ -108,18 +105,27 @@ namespace portablefs
 
 namespace fs = std::filesystem;
 
+enum SeekMode {
+    SeekSet = 0,
+    SeekCur = 1,
+    SeekEnd = 2
+};
+
 
 namespace portablefs
 {
     const std::string basePath = "./base";
+    using string = std::string;
+    
+    using std::to_string;
 
     class Dir
     {
     public:
         Dir() {}
         Dir(const String &path)
-            : it_(fs::directory_iterator(path).begin()),
+            : it_(fs::directory_iterator(path)),
-              end_(fs::directory_iterator(path).end()),
+              end_(fs::directory_iterator()),
               firstIncremented_(false)
         {
         }
@@ -136,22 +142,22 @@ namespace portablefs
 
         bool isFile()
         {
-            return file.is_regular_file();
+            return it_->is_regular_file();
         }
 
         bool isDirectory()
         {
-            return it_.is_directory();
+            return it_->is_directory();
         }
 
         String fileName() const
         {
-            return it_.path().filename().string();
+            return it_->path().filename().string();
         }
 
         size_t fileSize() const
         {
-            return it_.file_size();
+            return it_->file_size();
         }
 
     private:
@@ -165,28 +171,73 @@ namespace portablefs
         return Dir(path);
     }
 
+    class File 
+    {
+    public:
+        File(std::FILE * file) : file_(file) {}
+
+        bool seek(uint32_t pos, SeekMode mode) 
+        {
+            if(mode == SeekSet)
+                return std::fseek(file_, pos, SEEK_SET) == 0;
+            else if (mode == SeekCur)
+                return std::fseek(file_, pos, SEEK_CUR) == 0;
+            else 
+                return std::fseek(file_, pos, SEEK_END) == 0;
+        }
+
+        bool seek(uint32_t pos) { return std::fseek(file_, pos, SEEK_SET); }
+
+        size_t write(const uint8_t *buf, size_t size) {
+            return std::fwrite(buf, size, 1UL, file_);
+        }
+
+        size_t size() {
+            auto current_position = ftell(file_);
+            seek(0, SeekEnd);
+            auto end = ftell(file_);
+            seek(0, SeekSet);
+            auto begin = ftell(file_);
+            seek(current_position, SeekSet);
+            return end - begin;
+        }
+
+    private:
+        std::FILE * file_;
+    };
+
     inline File open(const char *name, const char *mode)
     {
-        if(mode == "r")
+        std::string modeStr;
-            return fopen()
+        if(modeStr.find("b") == std::string::npos)
-        return SPIFFS.open(name, mode);
+            modeStr += "b";
+        return File(std::fopen(name, modeStr.c_str()));
     }
 
     inline bool exists(const char *name)
     {
-        return SPIFFS.exists(name);
+        return fs::exists(fs::path(name));
     }
 
     inline bool remove(const char *name)
     {
-        return SPIFFS.remove(name);
+        return fs::remove(fs::path(name));
     }
 
     inline bool mkdir(const char *name)
     {
-        return SPIFFS.mkdir(name);
+        return fs::create_directory(fs::path(name));
     }    
 
+    //inline size_t totalBytes() {
+    //    return 1024 * 1024 * 4; // some dummy value
+    //}
+    //inline size_t usedBytes() {
+    //    // TODO: makes more sense to report total file size of all files in folder
+    //    return 1024; // some dummy value 
+    //}
+
+
 } // namespace portablefs
 
 #endif

firmware/lib/basic/MockSerial.h

@@ -2,6 +2,7 @@
 
 #include <iostream>
 
+#ifdef PLATFORM_NATIVE
 
 class SerialMock {
 public:
@@ -16,3 +17,6 @@ public:
 };
 
 static SerialMock Serial;
+
+
+#endif

firmware/lib/basic/PreferencesAbstraction.h

@@ -0,0 +1,36 @@
+#pragma once
+
+
+#ifdef PLATFORM_ESP32
+#include "Preferences.h"
+#endif 
+
+
+
+#ifdef PLATFORM_NATIVE
+
+#include <map>
+
+class Preferences
+{
+  public:
+    std::string getString(const char * key, const std::string & defaultValue) const {
+      std::string strKey(key);
+      const auto it = strings_.find(strKey);
+      return (it == string_.end()) ? defaultValue : *it;
+    }
+
+    size_t putString(const char * key, const char * value) {
+      strings_[std::string(key)] = std::string(value);
+      return std::strlen(value);
+    }
+
+    bool remove(const char * key) {
+      return strings_.remove(key); 
+    }
+
+  private:
+    std::map<std::string, std::string> strings_;
+};
+
+#endif // PLATFORM_NATIVE

firmware/lib/basic/TimeAbstraction.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#ifdef PLATFORM_ESP32
+#include <Arduino.h>
+#endif 
+
+#ifdef PLATFORM_NATIVE
+
+#include <chrono>
+
+inline unsigned long millis() {
+  static auto timeOfFirstCall = std::chrono::steady_clock::now();
+  const auto timePoint = std::chrono::steady_clock::now();
+  return std::chrono::duration_cast<std::chrono::milliseconds>(timePoint - timeOfFirstCall).count();
+}
+#endif

firmware/lib/logging/Logger.cpp

@@ -1,8 +1,9 @@
 #include "Logger.h"
+#include "AllocAbstraction.h"
 
 constexpr size_t LOG_SIZE = 1024 * 1024 * 2;
 
-static Logger *theLogger = nullptr;
+static Logger * theLogger = nullptr;
 
 Logger *Logger::getInstance()
 {
@@ -12,10 +13,8 @@ Logger *Logger::getInstance()
 void Logger::init()
 {
   theLogger = new Logger();
-  Serial.begin(115200);
+  #ifdef PLATFORM_ESP32
-  while (!Serial)
+  #endif
-  {
-  }
 }
 
 Logger::Logger()
@@ -23,24 +22,6 @@ Logger::Logger()
   data_ = (char *)heap_caps_malloc(LOG_SIZE, MALLOC_CAP_SPIRAM);
   totalSize_ = LOG_SIZE;
   currentSize_ = 0;
-
-  // write header placeholder
-  const auto millisPlaceholder = 0;
-  memcpy(&data_[currentSize_], &millisPlaceholder, sizeof(millisPlaceholder));
-  currentSize_ += sizeof(millisPlaceholder);
-
-  NtpTimeT ntpPlaceholder = 0;
-  memcpy(&data_[currentSize_], &ntpPlaceholder, sizeof(ntpPlaceholder));
-  currentSize_ += sizeof(ntpPlaceholder);
-}
-
-void Logger::setNtpTime(NtpTimeT ntpTime)
-{
-  auto data = getInstance()->data_;
-
-  const auto millisTime = millis();
-  memcpy(&data[0], &millisTime, sizeof(millisTime));
-  memcpy(&data[sizeof(millisTime)], &ntpTime, sizeof(ntpTime));
 }
 
 

firmware/lib/logging/Logger.h

@@ -1,35 +1,48 @@
 #pragma once
 
+#include "TimeAbstraction.h"
+
+#ifdef PLATFORM_ESP32
 #include "Arduino.h"
+#endif
+
+#ifdef PLATFORM_NATIVE
+#include <stdint.h>
+#include <string.h>
+#include <utility>
+#include <stdio.h>
+#endif
 
 #define LOG_INFO(...)                        \
   {                                          \
     Logger::getInstance()->log(__VA_ARGS__); \
   }
 
-#define LOG_TRACE(...)                       \
+/*#define LOG_TRACE(...)                       \
   {                                          \
     Logger::getInstance()->log(__VA_ARGS__); \
   }
+*/
+#define LOG_TRACE(...) {}
 
 #define LOG_WARNING(...)                     \
   {                                          \
     Logger::getInstance()->log(__VA_ARGS__); \
   }
 
+
+
 class Logger
 {
 public:
-  using NtpTimeT = unsigned long;
+  using TimeT = unsigned long;
-
-  static constexpr int HEADER_SIZE = sizeof(NtpTimeT) + sizeof(millis());
 
   ~Logger();
 
   template <class... Args>
   inline bool log(const char *formatStr, Args &&...args)
   {
-    const auto time = millis();
+    const TimeT time = millis();
 
     if (totalSize_ - currentSize_ <= sizeof(time))
       return false;
@@ -38,32 +51,64 @@ public:
     currentSize_ += sizeof(time);
 
     const auto spaceLeft = totalSize_ - currentSize_;
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-security"
     auto charsWritten = snprintf(&data_[currentSize_], spaceLeft, formatStr,
                                  std::forward<Args>(args)...);
-    Serial.println(&data_[currentSize_]);
+#pragma GCC diagnostic pop
 
-    if (charsWritten < spaceLeft)
+    Serial.println(&data_[currentSize_]);
+    if(charsWritten > 0)
     {
-      currentSize_ += charsWritten;
+      currentSize_ += charsWritten + 1; // + 1 for trailing zero
       return true;
     }
     else
       return false;
+    return true;
   }
 
   static Logger *getInstance();
   static void init();
-  static void setNtpTime(NtpTimeT time);
-
 
   const size_t totalSize() const { return totalSize_; }
   const size_t currentSize() const { return currentSize_; }
   const char * data() const { return data_; }
 
+  // Iteration
+  class iterator
+  {
+    public:
+      using TimeT = unsigned long;
+
+      iterator(const char * ptr) : current_position_(ptr) {}
+
+      TimeT time_millis() const {
+        return *reinterpret_cast<const TimeT*>(current_position_);
+      }
+
+      const char * message() const {
+        return current_position_ + sizeof(TimeT);
+      }
+
+      void operator++(){
+        current_position_ += sizeof(TimeT) + strlen(message()) + 1;
+      }
+
+      bool operator==(const iterator & o) const { return current_position_ == o.current_position_; }
+      bool operator!=(const iterator & o) const { return current_position_ != o.current_position_; }
+
+    private:
+      const char * current_position_;
+  };
+  iterator begin() const { return {data_}; }
+  iterator end()   const { return {data_ + currentSize_}; }
+
 private:
+
   Logger();
 
   char *data_;
-  size_t totalSize_;
+  size_t totalSize_ = 0;
-  size_t currentSize_;
+  size_t currentSize_ = 0;
 };

firmware/lib/scale/Scale.h

@@ -8,7 +8,7 @@ public:
     bool measure(uint16_t &measurementOut)
     {
         long value = hx711_.read_average(CONFIG_MEASUREMENT_AVG_COUNT);
-        LOG_TRACE("rv %ld\n", value);
+        LOG_TRACE("rv %ld", value);
         value -= offset_;
         
         if (value < 0)
@@ -28,7 +28,7 @@ public:
     {
         auto v1 = hx711_.read_average(3);
         offset_ = hx711_.read_average(numMeasurementsToAverage);
-        LOG_INFO("Init reading %ld, Tare offset %ld\n", v1, offset_);
+        LOG_INFO("Init reading %ld, Tare offset %ld", v1, offset_);
     }
 
     const long &offset() const { return offset_; }

firmware/lib/session/MockStorage.h

@@ -12,7 +12,7 @@ class VectorAdaptor {
 public:
     VectorAdaptor(std::vector<uint8_t> * v) : v_(v) {}
 
-    void write(const char *data, uint32_t size) {
+    void write(const uint8_t *data, uint32_t size) {
         v_->insert(v_->end(), data, data + size );
     }
 
@@ -38,7 +38,7 @@ public:
     FilePtrAdaptor(const FilePtrAdaptor &) = delete;
     void operator=(const FilePtrAdaptor &) = delete;
 
-    void write(const char *data, uint32_t size) {
+    void write(const uint8_t *data, uint32_t size) {
         fwrite(data, size, 1, fptr);
     }
 

firmware/lib/session/SessionManager.h

@@ -52,8 +52,8 @@ private:
     uint8_t tareAvgCount_;
     int valueRightShift_;
 
-    AutoStart<MeasurementT> autoStart_;
+    AutoStart<MeasurementType> autoStart_;
-    AutoStop<MeasurementT> autoStop_;
+    AutoStop<MeasurementType> autoStop_;
 };
 
 // ------------------------------------------------------------------------------------------------

firmware/lib/session/SimpleMeasurementSession.h

@@ -1,7 +1,9 @@
 #include "Dtypes.h"
 #include "SessionChunk.h"
 #include "FilesystemAbstraction.h"
+#include "AllocAbstraction.h"
 #include "Logger.h"
+#include "SwimTrackerConfig.h"
 
 
 template <typename Measurement_T, uint32_t MAX_SIZE>
@@ -31,8 +33,12 @@ public:
             new (chunk) ChunkT(); // placement new to init chunk
         }
         chunk->init(epochStartTime, 0);
+
+        // write header here - since this takes a long time later when measurement is running
+        writeToNewFile();
     }
 
+
     bool addPoint(Measurement_T measurement)
     {
         bool success = chunk->addPoint(measurement);
@@ -74,6 +80,19 @@ public:
     }
 
 private:
+    portablefs::string getFilename() {
+        return portablefs::string(CONFIG_DATA_PATH) + "/" + portablefs::to_string(chunk->getStartTime());
+    }
+
+    void writeToNewFile() {
+        LOG_INFO("Initializing file");
+        const auto filename = getFilename();
+        auto file = portablefs::open(filename.c_str(), "w");
+        StreamingMsgPackEncoder<portablefs::File> encoder(&file);
+        chunk->serialize(encoder);        
+        LOG_INFO("Initializing file done");
+    }
+
     void saveToFileSystem()
     {
         static const uint32_t arrayHeaderOffset = ChunkT::arrayHeaderOffset();
@@ -81,11 +100,11 @@ private:
 
         // todo: check this! free doesn't mean that the file writing actually works ok
         // use error codes of write instead? anyway: test it!
-        LOG_INFO("%ld saveToFileSystem start", millis());
+        LOG_INFO("saveToFileSystem start");
         deleteUntilBytesFree(CONFIG_SESSION_MAX_SIZE);
-        LOG_INFO("   %ld after deleteUntilBytesFree()", millis());
+        LOG_TRACE("after deleteUntilBytesFree()");
 
-        String filename = String(CONFIG_DATA_PATH) + "/" + String(chunk->getStartTime());
+        const auto filename = getFilename();
         if (portablefs::exists(filename.c_str()))
         {
             auto file = portablefs::open(filename.c_str(), "r+");
@@ -102,15 +121,15 @@ private:
         }
         else
         {
-            auto file = portablefs::open(filename.c_str(), "w");
+            LOG_INFO("Creating new session file, should have been done already");
-            StreamingMsgPackEncoder<portablefs::File> encoder(&file);
+            writeToNewFile();
-            chunk->serialize(encoder);
         }
-        LOG_INFO("   %ld saveToFileSystem done", millis());
+        LOG_INFO("saveToFileSystem done");
     }
 
     void deleteUntilBytesFree(size_t requiredSpace)
     {
+        #ifdef PLATFORM_ESP32
         auto freeBytes = portablefs::totalBytes() - portablefs::usedBytes();
         while (freeBytes < requiredSpace)
         {
@@ -140,6 +159,7 @@ private:
             assert(newFreeBytes > freeBytes);
             freeBytes = newFreeBytes;
         }
+        #endif // PLATFORM_ESP32
     }
 
     ChunkT *chunk;

firmware/lib/session/StreamingMsgPackEncoder.h

@@ -1,4 +1,5 @@
 #pragma once
+#include "Logger.h"
 
 template<typename T> struct TypeToMsgPackCode{};
 template<> struct TypeToMsgPackCode<uint8_t> { static const char CODE; };
@@ -52,7 +53,7 @@ public:
         else
         {
             size |= 0b10000000;
-            writer->write((byte*)(&size), 1);
+            writer->write((uint8_t*)(&size), 1);
         }
     }
 

firmware/lib/wifimanager/WifiManager.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Preferences.h"
+#include "PreferencesAbstraction.h"
 #include <WiFi.h>
 
 /**

firmware/platformio.ini

@@ -24,15 +24,18 @@ board_upload.flash_size = "16MB"
 #build_flags = -Wl,-Teagle.flash.2m1m.ld
 build_flags = -DPLATFORM_ESP32 -DBOARD_HAS_PSRAM -mfix-esp32-psram-cache-issue
 framework = arduino
-monitor_port = /dev/ttyUSB0
+monitor_port = /dev/ttyUSB1
-upload_port = /dev/ttyUSB0
+upload_port = /dev/ttyUSB1
 monitor_speed = 115200
+debug_tool = esp-prog
+#upload_protocol = esp-prog
+debug_init_break = tbreak setup
 lib_deps =
     NTPClient
     HX711@0.7.4
     ArduinoJson
     https://github.com/gilmaimon/ArduinoWebsockets.git
-src_filter = +<*> -<native_main.cpp>
+build_src_filter = +<*> -<native_main.cpp>
 board_build.partitions = partitions_custom.csv
 board_build.embed_txtfiles =
     certificate.pem
@@ -40,5 +43,7 @@ board_build.embed_txtfiles =
 [env:native]
 platform = native
 test_ignore = test_embedded
-build_flags = -g -DPLATFORM_NATIVE
+build_flags = -g -DPLATFORM_NATIVE  -std=c++17 -O0
-src_filter = +<*> -<firmware_main.cpp>
+build_src_filter = +<*> -<firmware_main.cpp> -<WifiManager.cpp> -<WifiAPI.cpp>
+lib_compat_mode = off
+

firmware/src/LoggingAPI.h

@@ -0,0 +1,56 @@
+#include "Dtypes.h"
+#include "MessageCodes.h"
+#include "Logger.h"
+#include "SwimTrackerConfig.h"
+
+#include <ArduinoWebsockets.h>
+#include <ArduinoJson.h>
+
+
+class LoggingAPI
+{
+  public:
+    void onClientConnect(websockets::WebsocketsClient &client, int /*clientId*/) {}
+
+    bool handleMessage(websockets::WebsocketsClient &client, MessageCode code, const char *payload, size_t size) {
+      switch(code) {
+        case MessageCode::LOG_STREAMING_START:
+          running_ = true;
+          if(firstCall_) {
+            const Logger * logger = Logger::getInstance();
+            lastEnd_ = logger->begin();
+            firstCall_ = false;
+          }
+          return true;
+        case MessageCode::LOG_STREAMING_STOP:
+          running_ = false;
+          return true;
+        default:
+          return false;
+      }
+    }
+
+    template <typename TServer>
+    void iteration(TServer &server)
+    {
+      if(running_)
+      {
+        const Logger * logger = Logger::getInstance();
+
+        const auto endIt = logger->end();
+        StaticJsonDocument<1024> data;
+        for(auto it = lastEnd_; it != endIt; ++it) {
+            data["time"] = it.time_millis();
+            data["msg"] = it.message();
+            server.template sendToAll<1024>(MessageCode::LOG_UPDATE, data);
+        }
+        lastEnd_ = endIt;
+      }
+    }
+
+  private:
+    bool running_ = false;
+    Logger::iterator lastEnd_ = Logger::iterator(nullptr);
+    bool firstCall_ = true;
+};
+

firmware/src/MessageCodes.h

@@ -15,6 +15,7 @@ enum class MessageCode : uint8_t
   WIFI_STATE_RESPONSE = 8,
   WIFI_SCAN_RESPONSE = 9,
   APP_LAYER_PING = 10,
+  LOG_UPDATE = 11,
 
   // from frontend to device
   START_SESSION = 128,
@@ -25,4 +26,6 @@ enum class MessageCode : uint8_t
   WIFI_STATE_SET = 133,
   WIFI_STATE_GET = 134,
   WIFI_TRIGGER_SCAN = 135,
+  LOG_STREAMING_START = 136,
+  LOG_STREAMING_STOP = 137
 };

firmware/src/SessionAPI.h

@@ -20,7 +20,7 @@ public:
       numSentMeasurements_[i] = 0;
   }
 
-  void onClientConnect(websockets::WebsocketsClient &client);
+  void onClientConnect(websockets::WebsocketsClient &client, int clientId);
   bool handleMessage(websockets::WebsocketsClient &client, MessageCode code, const char *payload, size_t size);
 
   template <typename TServer>
@@ -41,7 +41,7 @@ private:
 
 // sending message about current session
 template <typename T>
-void SessionAPI<T>::onClientConnect(websockets::WebsocketsClient &client)
+void SessionAPI<T>::onClientConnect(websockets::WebsocketsClient &client, int clientId)
 {
   // TODO write msgpack instead for consistency?
 
@@ -73,10 +73,12 @@ void SessionAPI<T>::onClientConnect(websockets::WebsocketsClient &client)
 
   assert(writeHead - msg == msgSize - sizeof(MeasurementT) * numMeasurements);
 
+  LOG_INFO("Start sending existing measurements");
   memcpy(writeHead, session.getDataPointer(), sizeof(MeasurementT) * numMeasurements);
   client.sendBinary(msg, msgSize);
-
+  numSentMeasurements_[clientId] = numMeasurements;
   free(msg);
+  LOG_INFO("Finished sending existing measurements");
 }
 
 template <typename T>
@@ -157,7 +159,7 @@ template <typename TServer>
 void SessionAPI<T>::sendNewDataMessages(TServer &server)
 {
 
-  constexpr size_t MAX_MEASUREMENTS_PER_MSG = 15;
+  constexpr size_t MAX_MEASUREMENTS_PER_MSG = 50;
   constexpr size_t WAIT_UNTIL_AT_LEAST_NUM_MEASUREMENTS = 1;
 
   // new data messages are the only messages not sent in msgpack format

firmware/src/WebsocketServer.h

@@ -3,8 +3,10 @@
 #include "Dtypes.h"
 #include "UserDB.h"
 #include "MessageCodes.h"
+#include "SwimTrackerConfig.h"
 
 #include <ArduinoWebsockets.h>
+#include <ArduinoJson.h>
 
 #include <type_traits>
 
@@ -48,15 +50,19 @@ public:
 
     if (server_.poll())
     {
-      LOG_INFO("new websocket connection, storing at pos %d - occupancy: ", nextFreeClient_);
       clients_[nextFreeClient_] = server_.accept();
       clients_[nextFreeClient_].onMessage(onMessage);
-      this->onClientConnectImpl(clients_[nextFreeClient_]);
+      this->onClientConnectImpl(clients_[nextFreeClient_], nextFreeClient_);
       nextFreeClient_ = (nextFreeClient_ + 1) % MAX_WEBSOCKET_CONNECTIONS;
 
+      if(MAX_WEBSOCKET_CONNECTIONS == 3) {
+        LOG_INFO("new websocket connection, storing at pos %d - occupancy: %d%d%d", nextFreeClient_, clients_[0].available(), clients_[1].available(), clients_[2].available());
+      } else {
+        LOG_INFO("new websocket connection, storing at pos %d - occupancy:", nextFreeClient_);
         for (int i = 0; i < MAX_WEBSOCKET_CONNECTIONS; ++i)
           LOG_INFO((clients_[i].available()) ? "x" : "o");
       }
+    }
 
     for (int i = 0; i < MAX_WEBSOCKET_CONNECTIONS; ++i)
       clients_[i].poll();
@@ -67,12 +73,13 @@ public:
   template <size_t bufferSize>
   void sendToAll(MessageCode msgCode, const JsonDocument &content)
   {
+    static_assert(sizeof(MessageCode) == 1);
     char buffer[bufferSize];
     buffer[0] = (char)(msgCode);
     size_t bytesWritten = serializeMsgPack(content, buffer + sizeof(msgCode), bufferSize - sizeof(msgCode));
     for (int i = 0; i < MAX_WEBSOCKET_CONNECTIONS; ++i)
       if (clients_[i].available())
-        clients_[i].sendBinary(buffer, bytesWritten);
+        clients_[i].sendBinary(buffer, bytesWritten + sizeof(msgCode));
   }
 
   void sendToAll(MessageCode msgCode, const JsonDocument &content)
@@ -121,13 +128,13 @@ private:
   bool handleMessageImpl(websockets::WebsocketsClient &, MessageCode, const char *, size_t) { return false; }
 
   template <size_t managerIdx = 0, typename std::enable_if<(managerIdx < std::tuple_size<ApiManagerTuple>::value), bool>::type = true>
-  void onClientConnectImpl(websockets::WebsocketsClient &client)
+  void onClientConnectImpl(websockets::WebsocketsClient &client, int clientId)
   {
-    std::get<managerIdx>(apiManagers_).onClientConnect(client);
+    std::get<managerIdx>(apiManagers_).onClientConnect(client, clientId);
-    onClientConnectImpl<managerIdx + 1>(client);
+    onClientConnectImpl<managerIdx + 1>(client, clientId);
   }
   template <size_t managerIdx, typename std::enable_if<managerIdx == std::tuple_size<ApiManagerTuple>::value, bool>::type = true>
-  void onClientConnectImpl(websockets::WebsocketsClient &client) {}
+  void onClientConnectImpl(websockets::WebsocketsClient &client, int clientId) {}
 
   // -- Members
 

firmware/src/WifiAPI.cpp

@@ -11,7 +11,7 @@ void WifiAPI::sendWifiState(websockets::WebsocketsClient &client)
   sendToClient<192>(client, MessageCode::WIFI_STATE_RESPONSE, data);
 }
 
-void WifiAPI::onClientConnect(websockets::WebsocketsClient &client)
+void WifiAPI::onClientConnect(websockets::WebsocketsClient &client, int /*clientId*/)
 {
   sendWifiState(client);
 }

firmware/src/WifiAPI.h

@@ -15,7 +15,7 @@ public:
   {
   }
 
-  void onClientConnect(websockets::WebsocketsClient &client);
+  void onClientConnect(websockets::WebsocketsClient &client, int clientId);
   bool handleMessage(websockets::WebsocketsClient &client, MessageCode code, const char *payload, size_t size);
 
   template <typename TServer>

firmware/src/firmware_main.cpp

@@ -25,6 +25,7 @@
 #include "WebsocketServer.h"
 #include "SessionAPI.h"
 #include "WifiAPI.h"
+#include "LoggingAPI.h"
 
 using Session_T = SimpleMeasurementSession<MeasurementT, CONFIG_SESSION_MAX_SIZE>;
 SessionManager<Session_T> sessionManager;
@@ -35,7 +36,7 @@ EspHttp espHttpServer;
 
 WifiManager wifiManager;
 
-auto apiTuple = std::make_tuple(SessionAPI<Session_T>(sessionManager), WifiAPI(wifiManager));
+auto apiTuple = std::make_tuple(SessionAPI<Session_T>(sessionManager), WifiAPI(wifiManager), LoggingAPI());
 WebsocketServer<decltype(apiTuple)> websocketServer(81, apiTuple);
 
 //WebsocketServer<Session_T> webSocketServer(sessionManager, userStorage, 81);
@@ -432,6 +433,10 @@ void mdnsSetup(const String &fullHostname)
 
 void setup()
 {
+    Serial.begin(115200);
+    while (!Serial)
+    {
+    }
     // Serial
     Logger::init();
 
@@ -460,7 +465,6 @@ void setup()
     mdnsSetup(configuredHostname);
 
     sessionManagerSetup();
-    Logger::setNtpTime(sessionManager.getNtpTime());
     sessionManager.tare();
 
     // HTTP & Websocket server

firmware/src/native_main.cpp

@@ -0,0 +1,23 @@
+#include "Logger.h"
+
+#include <iostream>
+#include <chrono>
+#include <thread>
+
+int main()
+{
+  Logger::getInstance()->init();
+  auto logger = Logger::getInstance();
+  using namespace std::chrono_literals;
+
+  LOG_INFO("Message1");
+  LOG_INFO("Message %d", 5);
+  std::this_thread::sleep_for(2000ms);
+  LOG_INFO("Message %s", "3");
+
+  for(auto it = logger->begin(); it != logger->end(); ++it)
+  {
+     std::cout << it.time_millis() << ": " << it.message() << std::endl;
+  }
+  return 0;
+}

firmware/test/test_common/sessiontest.cpp

@@ -1,6 +1,6 @@
 #include "Dtypes.h"
 #include "MockSerial.h"
-#include "MeasurementSession.h"
+#include "SimpleMeasurementSession.h"
 #include "MockStorage.h"
 #include <unity.h>
 #include <vector>
@@ -120,8 +120,8 @@ void testSessionChunkSerialization()
 
 
 void testSession() {
-    const uint32_t SESSION_SIZE = 128;
+    const uint32_t SESSION_SIZE = 1024;
-    typedef MeasurementSession<uint16_t, MockStorageReader, MockStorageWriter, SESSION_SIZE> MockSession;
+    using MockSession = SimpleMeasurementSession<uint16_t, SESSION_SIZE>;
 
     const uint32_t startTime = 194842;
     const uint_t fillSize = SESSION_SIZE * 4 + 7;
@@ -149,7 +149,7 @@ void testSession() {
 
 void testPartialSessionSerialization() {
     const uint32_t SESSION_SIZE = 1024*8 - 16 * sizeof(uint32_t);
-    typedef MeasurementSession<uint16_t, MockStorageReader, MockStorageWriter, SESSION_SIZE> MockSession;
+    using MockSession = SimpleMeasurementSession<uint16_t, SESSION_SIZE>;
 
     const uint32_t startTime = 194842;
     const uint_t fillSize = 4937 + 81;
@@ -191,7 +191,7 @@ void testPartialSessionSerialization() {
 
 void testPartialSessionSerializationEmptyArray() {
     const uint32_t SESSION_SIZE = 1024*8 - 16 * sizeof(uint32_t);
-    typedef MeasurementSession<uint16_t, MockStorageReader, MockStorageWriter, SESSION_SIZE> MockSession;
+    using MockSession = SimpleMeasurementSession<uint16_t, SESSION_SIZE>;
 
     const uint32_t startTime = 194842;
     const uint_t fillSize = 4937 + 81;

testsystem/.gitignore

@@ -0,0 +1,5 @@
+.pio
+.vscode/.browse.c_cpp.db*
+.vscode/c_cpp_properties.json
+.vscode/launch.json
+.vscode/ipch

testsystem/platformio.ini

@@ -0,0 +1,12 @@
+[platformio]
+default_envs = esp
+
+[env:esp]
+platform = espressif8266
+board = nodemcuv2
+framework = arduino
+monitor_port = /dev/ttyUSB0
+upload_port = /dev/ttyUSB0
+monitor_speed = 115200
+lib_deps =
+    ArduinoJson

testsystem/src/MotorControl.cpp

@@ -0,0 +1,76 @@
+#include "MotorControl.h"
+#include <Arduino.h>
+
+constexpr int PORT_FORWARD = 4;
+constexpr int PORT_BACKWARD = 16;
+constexpr int PORT_SPEED = 5;
+
+// PWM settings
+constexpr int PWM_FREQ = 20000;
+constexpr int PWM_CHANNEL = 0;
+constexpr int PWM_RESOLUTION = 255;
+
+void MotorControl::begin()
+{
+  pinMode(PORT_SPEED, OUTPUT);
+  pinMode(PORT_FORWARD, OUTPUT);
+  pinMode(PORT_BACKWARD, OUTPUT);
+
+  analogWriteRange(PWM_RESOLUTION);
+  analogWriteFreq(PWM_FREQ);
+  startTime_ = millis();
+}
+
+void MotorControl::addCmd(float signedSpeed, unsigned long duration)
+{
+  if (numCmds_ < MAX_MOTOR_COMMANDS)
+    motorCommands_[numCmds_++] = {int(signedSpeed * PWM_RESOLUTION), duration};
+  else
+    Serial.println("Too many commands");
+}
+
+float MotorControl::speed(int cmdIdx) const
+{
+  return float(motorCommands_[cmdIdx].directionAndSpeed) / float(PWM_RESOLUTION);
+}
+
+unsigned long MotorControl::duration(int cmdIdx) const
+{
+  return motorCommands_[cmdIdx].delayToNextCmdInMs;
+}
+
+void MotorControl::iteration()
+{
+  if (numCmds_ == 0)
+  {
+    analogWrite(PORT_SPEED, 0);
+    return;
+  }
+
+  unsigned long timeElapsed = millis() - startTime_;
+  while (timeElapsed > motorCommands_[currentCmd_].delayToNextCmdInMs)
+  {
+    const auto curCmdDuration = motorCommands_[currentCmd_].delayToNextCmdInMs;
+    timeElapsed -= curCmdDuration;
+    startTime_ += curCmdDuration;
+    currentCmd_ = (currentCmd_ + 1) % numCmds_;
+  }
+
+  const auto &curCmd = motorCommands_[currentCmd_];
+  const auto &nextCmd = motorCommands_[(currentCmd_ + 1) % numCmds_];
+  const float fraction = float(timeElapsed) / float(curCmd.delayToNextCmdInMs);
+  const int interpolated = int((1.f - fraction) * curCmd.directionAndSpeed + fraction * nextCmd.directionAndSpeed);
+
+  if (interpolated < 0)
+  {
+    analogWrite(PORT_SPEED, -interpolated);
+    digitalWrite(PORT_FORWARD, 0);
+    digitalWrite(PORT_BACKWARD, 1);
+  }
+  else
+  {
+    analogWrite(PORT_SPEED, interpolated);
+    digitalWrite(PORT_FORWARD, 1);
+    digitalWrite(PORT_BACKWARD, 0);
+  }
+}

testsystem/src/MotorControl.h

@@ -0,0 +1,31 @@
+
+
+class MotorControl
+{
+public:
+  void begin();
+  void clear()
+  {
+    numCmds_ = 0;
+    currentCmd_ = 0;
+  }
+  void addCmd(float signedSpeed, unsigned long duration);
+  void iteration();
+
+  int numCommands() const { return numCmds_; }
+  float speed(int cmdIdx) const;
+  unsigned long duration(int cmdIdx) const;
+
+private:
+  static constexpr int MAX_MOTOR_COMMANDS = 1024;
+  struct MotorCmd
+  {
+    int directionAndSpeed; // fraction of PWM_RESO
+    unsigned long delayToNextCmdInMs;
+  };
+
+  int currentCmd_ = 0;
+  int numCmds_ = 0;
+  MotorCmd motorCommands_[MAX_MOTOR_COMMANDS];
+  unsigned long startTime_ = 0;
+};

testsystem/src/main.cpp

@@ -0,0 +1,87 @@
+#include "MotorControl.h"
+
+#include <ESP8266WiFi.h>
+#include <ESP8266WebServer.h>
+#include <ArduinoJson.h>
+
+ESP8266WebServer server(80);
+MotorControl motorCtl;
+const char *webBuffer[1024 * 4];
+
+void wifiSetup()
+{
+  static const char *ssid = "WLAN";
+  static const char *password = "Bau3rWLAN";
+
+  WiFi.begin(ssid, password);
+  WiFi.hostname("swimtracker-tester");
+
+  while (WiFi.status() != WL_CONNECTED)
+  {
+    delay(500);
+    Serial.println("Waiting to connect...");
+  }
+  Serial.print("IP address: ");
+  Serial.println(WiFi.localIP());
+}
+
+void handleGetRoot()
+{
+  DynamicJsonDocument doc(motorCtl.numCommands() * 16 + 128);
+  JsonArray speeds = doc.createNestedArray("speeds");
+  JsonArray durations = doc.createNestedArray("durations");
+  for (int i = 0; i < motorCtl.numCommands(); ++i)
+  {
+    speeds.add(motorCtl.speed(i));
+    durations.add(motorCtl.duration(i));
+  }
+  size_t length = serializeJson(doc, webBuffer, sizeof(webBuffer));
+  server.send(200, "application/json", (const char *)webBuffer, length);
+}
+
+void handlePostRoot()
+{
+  String postBody = server.arg("plain");
+  DynamicJsonDocument doc(postBody.length() * 2 + 256);
+  DeserializationError error = deserializeJson(doc, postBody);
+  if (error || !doc.containsKey("speeds") || !doc.containsKey("durations"))
+    server.send(400, "application/json", "{ \"msg\" : \"Parsing error\" }");
+  else
+  {
+    JsonArray speeds = doc["speeds"].as<JsonArray>();
+    JsonArray durations = doc["durations"].as<JsonArray>();
+    if (speeds.size() != durations.size())
+      server.send(400, "application/json", "{ \"msg\" : \"Arrays have to have same length!\" }");
+    else
+    {
+      motorCtl.clear();
+      auto speedIt = speeds.begin();
+      auto durationIt = durations.begin();
+      for (; speedIt != speeds.end() && durationIt != durations.end(); ++durationIt, ++speedIt)
+      {
+        motorCtl.addCmd(speedIt->as<float>(), durationIt->as<unsigned long>());
+      }
+      server.send(200, "application/json", "{ \"msg\" : \"Ok\" }");
+    }
+  }
+}
+
+void setup()
+{
+  Serial.begin(115200);
+  wifiSetup();
+
+  server.on("/", HTTP_GET, handleGetRoot);
+  server.on("/", HTTP_POST, handlePostRoot);
+
+  server.begin(); //Start the server
+  Serial.println("Server listening");
+
+  motorCtl.begin();
+}
+
+void loop()
+{
+  server.handleClient(); //Handling of incoming client requests
+  motorCtl.iteration();
+}