ESP32 CAM Based Stereoscopic Camera with Serial Interface


Below is the code corresponding to this video. For ESP32 C-code, start with Arduino sketches for the ESP32 Camera board as the template; this will make life much easier as the sketches will include the dependencies... you just need to modify the .ino file with the new code.
Also, you will want to make sure you have the latest ESP32 files installed through the Arduino IDE boards manager - there may be some incompatibilities with the camera configuration options otherwise. Available from "https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json".

ESP32 C-Code for upload using Arduino IDE:

Master ESP32 Camera Code:
// Tinker Foundry
// Master camera device
// C-Code for ESP32 Stereo camera example - this is Master device code
// Connect to slave ESP32 camera device via Serial2 UART (RXD2 and TXD2) for device to device image data transfer
// Takes serial input from Serial interface to capture images and read back
#include <dummy.h>
#include "esp_camera.h"

// Select camera model
#define CAMERA_MODEL_WROVER_KIT // Has PSRAM
#include "camera_pins.h" // This must come after camera model has been selected

#define RXD2 32 // Serial2 interface pin definitions - connect to slave ESP32 device
#define TXD2 33 // Remember to connect TX to RX, and RX to TX

void setup() {
  Serial.begin(115200);
  Serial.setDebugOutput(true);
  Serial.println("Connecting...");

  camera_config_t config;
  config.ledc_channel = LEDC_CHANNEL_0;
  config.ledc_timer   = LEDC_TIMER_0;
  config.pin_d0       = Y2_GPIO_NUM;
  config.pin_d1       = Y3_GPIO_NUM;
  config.pin_d2       = Y4_GPIO_NUM;
  config.pin_d3       = Y5_GPIO_NUM;
  config.pin_d4       = Y6_GPIO_NUM;
  config.pin_d5       = Y7_GPIO_NUM;
  config.pin_d6       = Y8_GPIO_NUM;
  config.pin_d7       = Y9_GPIO_NUM;
  config.pin_xclk     = XCLK_GPIO_NUM;
  config.pin_pclk     = PCLK_GPIO_NUM;
  config.pin_vsync    = VSYNC_GPIO_NUM;
  config.pin_href     = HREF_GPIO_NUM;
  config.pin_sscb_sda = SIOD_GPIO_NUM;
  config.pin_sscb_scl = SIOC_GPIO_NUM;
  config.pin_pwdn     = PWDN_GPIO_NUM;
  config.pin_reset    = RESET_GPIO_NUM;
  config.xclk_freq_hz = 20000000;
  config.pixel_format = PIXFORMAT_GRAYSCALE;
  config.grab_mode    = CAMERA_GRAB_LATEST; //CAMERA_GRAB_WHEN_EMPTY; //
  config.frame_size   = FRAMESIZE_QVGA; //FRAMESIZE_96X96; //FRAMESIZE_QQVGA; //FRAMESIZE_SVGA;
  config.jpeg_quality = 12;
  config.fb_count     = 1;
  config.fb_location  = CAMERA_FB_IN_DRAM;
 
  // camera initialization
  esp_err_t err = esp_camera_init(&config);
  if (err != ESP_OK) {
    Serial.printf("Msg: Camera init failed with error 0x%x", err);
    return;
  }

  sensor_t * s = esp_camera_sensor_get();
  // drop down frame size for higher initial frame rate
  s->set_framesize(s, FRAMESIZE_QVGA);
  Serial.print("Msg: Camera Ready! Use 'http://");

  Serial.println("Setting up serial connection to Camera 2");
  Serial2.begin(115200, SERIAL_8N1, RXD2, TXD2); // set-up serial communication with the slave ESP32 device
  Serial.println("Done Serial2 setup");

}

// Main Loop
void loop() {
  while (Serial.available() == 0){} // Wait for serial input from console/PC
  String testStr = Serial.readString();
  testStr.trim();
  if (testStr == "CaptureMaster") // command to capture image from master camera
  {
    // now capture an image on this camera
    camera_fb_t * fb = NULL;
    fb = esp_camera_fb_get(); // get image... part of work-around to get latest image
    esp_camera_fb_return(fb); // return fb... part of work-around to get latest image
       
    fb = NULL;
    fb = esp_camera_fb_get(); // get fresh image
    size_t fbsize = fb->len;
    size_t fbwidth = fb->width;
    size_t fbheight = fb->height;

    // ----------------------------------------------------------------------------------------------------------------------
    // Now make the slave camera capture a frame so it's synchronized to take an image as close to master device as possible
    // ----------------------------------------------------------------------------------------------------------------------
    Serial2.print("capimg"); // This tells the slave to capture an image in the frame buffer
    while (Serial2.available() == 0){}
    String testStr2 = Serial2.readString();
    testStr2.trim();
    if (testStr2 != "OK") // Slave got the message and should be capturing the image
    {
      Serial.println("ERROR - Slave Camera Did not ACK the capture request");
    }

    // Now transfer master camera data to PC
    Serial.println("START");
    Serial.print(fbwidth);  // indicate the size of data (in bytes) to come...
    Serial.print("x");
    Serial.println(fbheight); // indicate the size of data (in bytes) to come...
    // push out master camera image data through main Serial port
    for (int i = 0; i < fbsize; i++)
    {
      Serial.println(fb->buf[i]); // write one byte at a time
    }
    Serial.println("END");
    esp_camera_fb_return(fb);
   
  } else if (testStr == "CaptureSlave") // With this command, we get the capture frame data from the slave ESP32 camera device, which has already been grabbed and waiting
  {
    Serial2.print("readimg"); // This tells the slave to send the image data from frame buffer
    String SlvString;

    Serial.println("START");
    Serial.print("320");  // indicate the size of data (in bytes) to come...
    Serial.print("x");
    Serial.println("240"); // indicate the size of data (in bytes) to come...

    while (Serial2.available() == 0){}
    for (int pix_idx = 0; pix_idx < 76800; pix_idx++) // 320x240 (hard-coded for now) - in this loop, we get data one byte at a time from second camera
    {
      while (Serial2.available() == 0){} // wait for slave device to send next data
      SlvString = Serial2.read();
      Serial.println(SlvString);
    }

    Serial.println("END");
  }

}



Slave ESP32 Camera Code:
// Tinker Foundry
// Slave Camera device
// C-Code for ESP32 Stereo camera example - this is Slave device code
// Takes command via serial interface to capture image and return data from frame buffer
#include "esp_camera.h"

// Select camera model
#define CAMERA_MODEL_WROVER_KIT // Has PSRAM
#include "camera_pins.h" // This must come after camera model has been selected

#define RXD2 32 // Serial2 interface pin definitions - connect to master ESP32 device
#define TXD2 33 // Remember to connect TX to RX, and RX to TX

camera_fb_t * fb = NULL;
size_t fbsize;

void setup() {
  Serial.begin(115200);
  Serial.setDebugOutput(true);
  Serial.println("Connecting...");

  // set up second serial/UART interface that connects to master ESP32 board
  Serial2.begin(115200, SERIAL_8N1, RXD2, TXD2);

  // configure the camera exactly the same as the master device
  camera_config_t config;
  config.ledc_channel = LEDC_CHANNEL_0;
  config.ledc_timer   = LEDC_TIMER_0;
  config.pin_d0       = Y2_GPIO_NUM;
  config.pin_d1       = Y3_GPIO_NUM;
  config.pin_d2       = Y4_GPIO_NUM;
  config.pin_d3       = Y5_GPIO_NUM;
  config.pin_d4       = Y6_GPIO_NUM;
  config.pin_d5       = Y7_GPIO_NUM;
  config.pin_d6       = Y8_GPIO_NUM;
  config.pin_d7       = Y9_GPIO_NUM;
  config.pin_xclk     = XCLK_GPIO_NUM;
  config.pin_pclk     = PCLK_GPIO_NUM;
  config.pin_vsync    = VSYNC_GPIO_NUM;
  config.pin_href     = HREF_GPIO_NUM;
  config.pin_sscb_sda = SIOD_GPIO_NUM;
  config.pin_sscb_scl = SIOC_GPIO_NUM;
  config.pin_pwdn     = PWDN_GPIO_NUM;
  config.pin_reset    = RESET_GPIO_NUM;
  config.xclk_freq_hz = 20000000;
  config.pixel_format = PIXFORMAT_GRAYSCALE;
  config.grab_mode    = CAMERA_GRAB_LATEST; //CAMERA_GRAB_WHEN_EMPTY; //
  config.frame_size   = FRAMESIZE_QVGA; //FRAMESIZE_96X96; //FRAMESIZE_QQVGA; //FRAMESIZE_SVGA;
  config.jpeg_quality = 12;
  config.fb_count     = 1;
  config.fb_location  = CAMERA_FB_IN_DRAM;
 
  // camera init
  esp_err_t err = esp_camera_init(&config);
  if (err != ESP_OK) {
    Serial.printf("Msg: Camera init failed with error 0x%x", err);
    return;
  }

  sensor_t * s = esp_camera_sensor_get();
  // drop down frame size for higher initial frame rate
  s->set_framesize(s, FRAMESIZE_QVGA);
}

void loop() {
  while (Serial2.available() == 0){} // wait for command from master ESP32 camera device
  String testStr = Serial2.readString();
  Serial.println("Received message from master");
  testStr.trim();
  Serial.println(testStr);
  if (testStr == "capimg") // this command triggers an image capture
  {
    delay(50);
    // send an acknowledge back to master
    Serial2.print("OK");
    Serial.println("Slave got capimg command - capturing image");

    fb = NULL;
    fb = esp_camera_fb_get(); // get image... part of work-around to get latest image
    esp_camera_fb_return(fb); // return fb... part of work-around to get latest image
       
    fb = NULL;
    fb = esp_camera_fb_get(); // get fresh image
    fbsize = fb->len;

    Serial.println("Slave done image capture");
  } else if (testStr == "readimg") // this command initiates transfer of image data
  {
    Serial.println("Slave received transfer image command from master");

    int row_idx = 0;
    int col_idx = 0;

    if (!fb)
    {
      Serial.println("Msg: No image - failed");
    } else
    {
      Serial.println("Slave Start Pixel Transfer to Master");
      size_t fbsize   = fb->len;
      size_t fbwidth  = fb->width;
      size_t fbheight = fb->height;
     
      for (int pixel_idx = 0; pixel_idx < fbsize; pixel_idx++) // this is for grayscale images only - 1 component per pixel
      {
         Serial2.write(fb->buf[pixel_idx]); // write next byte to master
         delay(1); // This slows down the interface so it doesn't overrun the master
      }
      Serial.println("Slave transfer to master done.");
      esp_camera_fb_return(fb); // free the buffer once we've read out the data
    }      
  } else
  {
    Serial.println("No valid command received from master.");
  }

}



Python Code to interface with ESP32 Master and Capture Images:
# Tinker Foundry
# Stereo_serial_reader.py
# Stereoscopic ESP32 Camera Project
# This code will initiate a frame by frame grab and write each picture into separate files (master and slave)
# Serial data reader from ESP32 serial output
# Corresponding Arduino sketch waits for serial command of 'CaptureMaster' to initiate a single camera frame grab and output of pixels from master camera
# CaptureMaster command also triggers a picture grab on the slave ESP32 camera, so that the images are synchronized in time as close as possible
# Serial command of "CaptureSlave" initiates transfer of captured image data from the slave device (through the master)
# dependency on pySerial (pip install pySerial)
import serial
ser = serial.Serial('COM4', 115200) # change 'COM4' to match whatever serial port your ESP32 board is connected to
ser.flushInput()

# Capture a series of images and write to file
for img_idx in range (1,2): # hard code 1 set(s) of images to capture for now... can do more sequential captures, but it's slow...
    # --------------------
    # Master Camera Read
    # --------------------
    print("Iteration start")
    print("Read Master Camera")
    ser.flushInput()
    ser.flushOutput()
    imgfileM = "imgdump" + str(img_idx) + "Master.txt"
    fM = open(imgfileM, 'w', encoding='UTF8', newline='')
    print("About to make file: " + imgfileM)
    ser.write(b'CaptureMaster\n')
    vld_dat = 0;

    while True:
        try:
            ser_bytes = ser.readline()
            dataString = ser_bytes.decode('utf-8')
            data = dataString[0:][:-2]
            if (data == "START"):
               print("START")
               vld_dat = 1;
            if ((data != "START") & (data != "END") & (vld_dat > 0)):
               fM.write(data)
               fM.write("\n")
            if (data == "END"):
               print("END...")
               break
        except KeyboardInterrupt:
            print("Keyboard Interrupt")
            break
     
    # --------------------
    # Slave Camera Read
    # --------------------
    print("Read Slave Camera")
    imgfileS = "imgdump" + str(img_idx) + "Slave.txt"
    fS = open(imgfileS, 'w', encoding='UTF8', newline='')
    print("About to make file: " + imgfileS)
    ser.write(b'CaptureSlave\n')
    vld_dat = 0;

    while True:
        try:
            ser_bytes = ser.readline()
            dataString = ser_bytes.decode('utf-8')
            data = dataString[0:][:-2]
            if (data == "START"):
              print("START")
              vld_dat = 1;
            if ((data != "START") & (data != "END") & (vld_dat > 0)):
               fS.write(data)
               fS.write("\n")
            if (data == "END"):
               print("END...")
               break
        except KeyboardInterrupt:
            print("Keyboard Interrupt")
            break

    fM.close()
    fS.close()
    print("Iteration end")



Python Code to read files and Display Them:
# Tinker Foundry
# StereoReadFile.py
# Stereoscopic ESP32 Camera Project
# Reads camera image files generated by Stereo_serial_reader from ESP32 Camera stereoscopic read and displays them
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image, ImageOps

# Loop will read files generated by the stereoscopic camera serial reader and display them
for img_idx in range (1,2): # hard-code 1 set of image(s) to read for now...
    print("reading files")

    # Start with the Master camera image
    imgfile = "imgdump" + str(img_idx) + "Master.txt"
    print(imgfile)
    f = open(imgfile, 'r')
    Lines = f.readlines()
    firstLine = Lines[0]
    resolution = firstLine.split("x")
    width = int(resolution[0])
    height = int(resolution[1])

    print(width)  # print out image width and height - will be used for pixel array initialization
    print(height)
    pixel_array = np.zeros([height, width], dtype=np.uint8)

    # Go through all the pixel data and store in the pixel array
    count = 0
    skipped_first = 0
    for line in Lines:
        cleanLine = line.strip()
        OneLine = cleanLine #.split(",")
        if (skipped_first > 0):
           single_pix = int(OneLine)
           #print(single_pix)
           y = count // width
           x = count % width
           pixel_array[y,x] = single_pix    
           count+=1
        skipped_first = 1

    # show the first image
    FinalImg = Image.fromarray(pixel_array)
    FinalImg.show()

    # Next read the Slave camera image - same steps as above
    imgfile = "imgdump" + str(img_idx) + "Slave.txt"
    print(imgfile)
    f = open(imgfile, 'r')
    Lines = f.readlines()
    firstLine = Lines[0]
    resolution = firstLine.split("x")
    width = int(resolution[0])
    height = int(resolution[1])

    print(width)
    print(height)

    pixel_array = np.zeros([height, width], dtype=np.uint8)

    count = 0
    skipped_first = 0
    for line in Lines:
        cleanLine = line.strip()
        OneLine = cleanLine #.split(",")
        if (skipped_first > 0):
           single_pix = int(OneLine)
           #print(single_pix)
           y = count // width
           x = count % width
           pixel_array[y,x] = single_pix    
           count+=1
        skipped_first = 1

    FinalImg = Image.fromarray(pixel_array)
    FinalImg.show()


Comments

Post a Comment

Popular posts from this blog

ESP32 Based Pulse-Oximeter using MAX30102

Image
Code for Arduino IDE used in the video is listed below. It targets the Wemos D1 R32 ESP32 board, but can be modified to suit. // Tinker Foundry // Accompanying video: https://youtu.be/xjwSKy6jzTI // Code for ESP32 to interface with MAX30102 breakout board and report blood oxygen level // Refer to https://github.com/DKARDU/bloodoxygen - below code is simplified version // https://www.analog.com/media/en/technical-documentation/data-sheets/MAX30102.pdf // Connections from WEMOS D1 R32 board to MAX30102 Breakout Board as follows: //  SCL (ESP32) to SCL (breakout board) //  SDA (ESP32) to SDA (breakout board) //  3V3 (ESP32) to VIN (breakout board) //  GND (ESP32) to GND (breakout board) #include <Wire.h> #include "MAX30105.h" //sparkfun MAX3010X library MAX30105 particleSensor; double avered    = 0 ; double aveir     = 0 ; double sumirrms  = 0 ; double sumredrms = 0 ; int    i         = 0 ; int    Num       = 100 ;  // calculate SpO2 by this sampling interval int
Read more

ESP32 Web Sockets - Transferring Image Raw Data

Image
Code listings... For the ESP32 C-code, please start off with a sketch from any existing example for ESP32 CAM and use that as your starting point... it will include app_httpd.cpp and camera_pins.h files. CameraWebSocket.ino // refer to https://shawnhymel.com/1675/arduino-websocket-server-using-an-esp32/ // To make sure WebSocketServer.h is available: //    Sketch > Include Library > Manage Libraries //    Search for "WebSockets Sattler' and install the library named "WebSockets by Markus Sattler" // // WARNING!!! PSRAM IC required for UXGA resolution and high JPEG quality //            Ensure ESP32 Wrover Module or other board with PSRAM is selected //            Partial images will be transmitted if image exceeds buffer size // #include "esp_camera.h" #include <WiFi.h> #include <WebSocketsServer.h> // Select camera model #define CAMERA_MODEL_WROVER_KIT // Has PSRAM #include "camera_pins.h" // This must come after camera m
Read more

Heart Rate Measurement with ESP32 and Pulse Oximeter Module using FFT

Image
 Code listing used in video: // Tinker Foundry // Heart Rate Measurement with the MAX30102 breakout board using FFT // Connections from WEMOS D1 R32 board to MAX30102 Breakout Board as follows: //  SCL (ESP32) to SCL (breakout board) //  SDA (ESP32) to SDA (breakout board) //  3V3 (ESP32) to VIN (breakout board) //  GND (ESP32) to GND (breakout board) #include <Wire.h> #include "MAX30105.h"     // sparkfun MAX3010X library MAX30105 particleSensor; #include "arduinoFFT.h" arduinoFFT FFT; double avered       = 0 ; double aveir        = 0 ; double sumirrms     = 0 ; double sumredrms    = 0 ; int    i            = 0 ; int    Num          = 100 ;  // calculate SpO2 by this sampling interval int    Temperature; int    temp; float  ESpO2;               // initial value of estimated SpO2 double FSpO2        = 0.7 ;  // filter factor for estimated SpO2 double frate        = 0.95 ; // low pass filter for IR/red LED value to eliminate AC component #def
Read more