Home Water Flow/Leak Detection using ESP32 Camera Module - Part II

#include "esp_camera.h"
//#include <WiFi.h>
//
// WARNING!!! PSRAM IC required for UXGA resolution and high JPEG quality
//            Ensure ESP32 Wrover Module or other board with PSRAM is selected
//            Partial images will be transmitted if image exceeds buffer size
//
 
// Select camera model
#define CAMERA_MODEL_WROVER_KIT // Has PSRAM
 
#include "camera_pins.h"
 
// theta and rho arrays for Hough transform
float thetas[90];
int rs[270];
int accumulator[270][90]; // accumulator for Hough transform for line detection
 
int last_theta = 0;
int go = 1;
 
// function to convert RGB to HSV
void rgb2hsv (int r, int g, int b, int *h, int *s, int *v)
{
  float rf = float(r)/255;
  float gf = float(g)/255;
  float bf = float(b)/255;
 
  float mx;
  float mn;
  float df;
 
  // find max
  if ((rf > gf) && (rf > bf))
  {
    mx = rf;
  } else if ((gf > rf) && (gf > bf))
  {
    mx = gf;
  } else if ((bf > rf) && (bf > gf))
  {
    mx = bf;
  } else
  {
    mx = rf;
  }
  // find min
  if ((rf < gf) && (rf < bf))
  {
    mn = rf;
  } else if ((gf < rf) && (gf < bf))
  {
    mn = gf;
  } else if ((bf < rf) && (bf < gf))
  {
    mn = bf;
  } else
  {
    mn = rf;
  }
 
  df = mx - mn;
 
  // H
  if (mx == mn)
  {
    *h = 0;
  } else if (mx == rf)
  {
    *h = int(60 * ((gf-bf)/df) + 360) % 360;
  } else if (mx == gf)
  {
    *h = int(60 * ((bf-rf)/df) + 120) % 360;
  } else if (mx == bf)
  {
    *h = int(60 * ((rf-gf)/df) + 240) % 360;
  }
  // S
  if (mx == 0)
  {
    *s = 0;
  } else
  {
    *s = int(df/mx*100);    
  }  
  // V
  *v = int(mx*100);
}
 
void setup() {
  Serial.begin(115200);
  Serial.setDebugOutput(true);
  Serial.println();
 
  camera_config_t config;
  config.ledc_channel = LEDC_CHANNEL_0;
  config.ledc_timer = LEDC_TIMER_0;
  config.pin_d0 = Y2_GPIO_NUM;
  config.pin_d1 = Y3_GPIO_NUM;
  config.pin_d2 = Y4_GPIO_NUM;
  config.pin_d3 = Y5_GPIO_NUM;
  config.pin_d4 = Y6_GPIO_NUM;
  config.pin_d5 = Y7_GPIO_NUM;
  config.pin_d6 = Y8_GPIO_NUM;
  config.pin_d7 = Y9_GPIO_NUM;
  config.pin_xclk = XCLK_GPIO_NUM;
  config.pin_pclk = PCLK_GPIO_NUM;
  config.pin_vsync = VSYNC_GPIO_NUM;
  config.pin_href = HREF_GPIO_NUM;
  config.pin_sscb_sda = SIOD_GPIO_NUM;
  config.pin_sscb_scl = SIOC_GPIO_NUM;
  config.pin_pwdn = PWDN_GPIO_NUM;
  config.pin_reset = RESET_GPIO_NUM;
  config.xclk_freq_hz = 20000000;
  config.pixel_format = PIXFORMAT_RGB888; // RGB888 format capture (change from original WebServer sketch)
 
  config.frame_size = FRAMESIZE_96X96; // Only capture 96x96 pictures //FRAMESIZE_QQVGA; //FRAMESIZE_SVGA;
  config.jpeg_quality = 12;
  config.fb_count = 1;
 
  // camera init
  esp_err_t err = esp_camera_init(&config);
  if (err != ESP_OK) {
    Serial.printf("Msg: Camera init failed with error 0x%x", err);
    return;
  }
 
  sensor_t * s = esp_camera_sensor_get();
  // drop down frame size for higher initial frame rate
  s->set_framesize(s, FRAMESIZE_96X96);
 
/* For now, comment out Wifi stuff */
 
}
 
void loop() {
 
  //Serial.println("Loop start...");
 
  camera_fb_t * fb;
 
  // Setup thetas array by converting range of degrees to radians
  // radians = (degrees * 71) / 4068
  // degrees = (radians * 4068) / 71
  // We don't have enough memory to store a value at 1 degree resolution so use every 2 degrees
  int tidx = 0;
  for (int t = -90; t < 90; t+=2)
  {
     thetas[tidx] = (float(t)*71)/4068;
     tidx+=1;
  }
 
  // setup rhos array from range -135 to +135
  int ridx = 0;
  for (int r = -135; r <= 135; r++)
  {
    rs[ridx] = r;
    ridx+=1;
  }
 
  // initialize accumulator to all zeros
  for (int theta_idx = 0; theta_idx < 90; theta_idx++)
  {
    for (int rho_idx = 0; rho_idx < 270; rho_idx++)
    {
      accumulator[rho_idx][theta_idx] = 0;
    }
  }
 
  // Capture stream of frame buffer data
  if (go == 1)
  {
    esp_camera_fb_return(fb); // return frame buffer contents if any
    fb = esp_camera_fb_get(); // capture frame
    delay(1000);
 
    if (!fb)
    {
      Serial.println("Msg: Camera capture failed");
      ESP.restart();
    } else
    {
      Serial.println("START");
      int fbidx = 0;
      for (int pixel_idx = 0; pixel_idx < 9216; pixel_idx++) // fixed 96x96 frame grab
      {
         int R,G,B;
         fbidx = pixel_idx*3;
         B = fb->buf[fbidx];
         //Serial.print(fb->buf[fbidx]); //B
         //Serial.print(",");
         G = fb->buf[fbidx+1];
         //Serial.print(fb->buf[fbidx+1]); //G
         //Serial.print(",");
         R = fb->buf[fbidx+2];
         //Serial.println(fb->buf[fbidx+2]); //R
 
         // --- Hough transform stuff ---
         int xc = pixel_idx % 96; // column
         int yc = pixel_idx / 96; // row
         
         // We want to convert RGB to pixels to HSV color space
         // This easily allows us to select and segment portions of the image by color
         int H,S,V;
         rgb2hsv(R,G,B,&H,&S,&V);
                 
         // Based on HSV thresholding, select red portions - corresponds to our red meter needle
         // For each red needle pixel, apply Hough transform...
         // Basically for each pixel, we determine all possible lines it could be part of and increment the accumulator accordingly
         // The line represented by rho and theta in the accumulator with the most "hits" should correspond to the line
         // formed by the red needle
         //if ((H<30) & (S>10) & (V>35))
         if ((H<30) & (S>10) & (V>40))
         {
           // this is a needle pixel
           for (int k=0; k < 90; k++) // sweep for each angle in theta array
           {
             float r = xc*cos(thetas[k]) + yc*sin(thetas[k]);
             accumulator[int(r) + 135][k] += 1;
           }
         }
      }
 
      Serial.println("END");
 
      // now we've processed the entire frame, look for the maximum in the accumulator
      int max_theta_idx = 0;
      int max_rho_idx = 0;
      int max_accval = 0;
      for (int theta_idx = 0; theta_idx < 90; theta_idx++)
      {
        for (int rho_idx = 0; rho_idx < 270; rho_idx++)
        {
          if (accumulator[rho_idx][theta_idx] > max_accval)
          {
            max_accval = accumulator[rho_idx][theta_idx];
            max_rho_idx = rho_idx;
            max_theta_idx = theta_idx;
          }
        }
      }
 
      int rho_val = int(rs[max_rho_idx]);
      float theta_val = (thetas[max_theta_idx]);
      int theta_deg = int((theta_val * 4068) / 71);
      Serial.print("rho: ");
      Serial.print(rho_val);
      Serial.print(" Theta: ");
      Serial.println(theta_deg);
 
      Serial.print("Previous theta: ");
      Serial.println(last_theta);
 
      // If the current theta angle of the line is sufficiently different from the previous one we computed,
      // then we say the meter needle is moving and Flow is Detected
      // We are ignoring rho values here... should be OK
      if (abs(theta_deg - last_theta) > 10)
      {
        Serial.println("---- Flow Detected -----");
      } else
      {
        Serial.println("---- No Flow ----");
               
      }
      last_theta = theta_deg; // now make current theta the last theta for next iteration
      Serial.println("----------------");
 
      esp_camera_fb_return(fb);
     
    }      
    go = 0;
  } else
  {
    //Serial.println("No capture.");
    go = 1;
  }
  delay(5000);
}