SRL Archive - Research & Development Lounge

I've been reading about and messing with basic motion detection. It's pretty darn fun!

I figured I would share a little test harness so anyone else can play with it. It runs fun ghost videos of the motion detected in the sequence you gather.

I bet The Mayor can find a Gelatinous Abomination in this ehehe:


Run it through posterize 88 and the river goes away:



Make a test folder to hold screen snaps. Log into runescape and run the first script. It will grab a few hundred snaps.

Then you can quit runescape and use the second script to see what it looks like if you compare the differences between frame x and x+1. Messing with it to compare to frame x-5, x-10 is good too.

Data capture script:

program grabFrames;
{$DEFINE SMART}
{$include_once srl-6/srl.simba}

var
   i: integer;
   b1, b2: integer;
   outpath, fname: string;
   t: ttimemarker;
   w, h: integer;

begin
   inputquery('Original', 'Enter output folder:', outpath);

   smartPlugins := ['d3d9.dll'];
   SetupSRL();

   writeln('Gathering in 5 seconds...');
   sleep(5000);

   writeln('Gathering');
   t.start();
   for i := 0 to 399 do
   begin
      b1 := bitmapfromclient(mainscreen.getbounds());
      getbitmapsize(b1, w, h);
      b2 := copybitmap(b1);
      
      // xx := greyscalebitmap(b2);  much faster with just grey

      getbitmapsize(b2, w, h);
      fname := outPath + '\snap.' + inttostr(i) + '.bmp';
      savebitmap(b2, fname);
      freebitmaps([b1, b2]);
   end;
   writeln('Done.  Time: ', t.getTime(), ' fps=', 100/(t.getTime()/1000));
end.

Analysis script:

program showMotion;
{$DEFINE SMART}
{$include_once srl-6/srl.simba} 

procedure greyBmp(var bmp: integer);
var b, g: integer;
begin
   b := copybitmap(bmp);
   g := greyscalebitmap(b);
   freebitmaps([bmp, b]);
   bmp := g;
end;

procedure posterizeBmp(var bmp: integer; n: integer);
var b, g: integer;
begin
   b := copybitmap(bmp);
   g := posterizebitmap(b, n);
   freebitmaps([bmp, b]);
   bmp := g;
end;

procedure desatBmp(var bmp: integer);
var b, g: integer;
begin
   b := copybitmap(bmp);
   g := desaturate(b);
   freebitmaps([bmp, b]);
   bmp := g;
end;
   
function bmpDiff(image1, image2: integer; doGrey: boolean = false;
   doDesat: boolean = false; doPost: boolean = false; pval: integer = 0 ): integer;

var
   w, h, i, j: integer;  
   work1, work2, diffBmp, c: integer;
   
begin
   getbitmapsize(image1, w, h);
   diffBmp := createbitmap(w,h);
   work1 := copybitmap(image1);
   work2 := copybitmap(image2);

   if (doGrey) then
   begin
      greyBmp(work1);
      greyBmp(work2);
   end;
   if (doDesat) then
   begin
      desatBmp(work1);
      desatBmp(work2);
   end;
   if (doPost) then
   begin
      posterizeBmp(work1, pval);
      posterizeBmp(work2, pval);
   end;
   
   for i := 0 to w-1 do
      for j := 0 to h-1 do
      begin
         c := fastgetpixel(work2, i, j);
         if (fastgetpixel(work1, i, j) <> c) then
            fastsetpixel(diffBmp, i, j, c);
      end;
      
   result := diffBmp;
   freebitmaps([work1, work2]);
end;

{ --------------- main -------------------}

var
   i, w, h, ans, pval: integer;
   inpath, fname, pstr: string;
   bmps: array[0..399] of integer;
   outBmps: array[0..398] of integer;
   t: TTimeMarker;
   doGrey, doDesat, doPost: boolean;
  
begin
   inputquery('Original', 'Enter input folder:', inpath);
   ans := messagebox('Greyscale?', '', 4);
   if (ans = 6) then doGrey := true;
   ans := messagebox('Desaturate?', '', 4);
   if (ans = 6) then doDesat := true;
   ans := messagebox('Posterize?', '', 4);
   if (ans = 6) then doPost := true;
   if (doPost) then
   begin
      inputquery('Posterize', 'Enter number for posterize (0-255):', pstr);
      pval := strtoint(pstr);
   end;
      
   t.start();
   for i := 0 to 399 do
   begin
      writeln('. processing ', i);
      fname := inpath + '\snap.' + inttostr(i) + '.bmp';
      bmps[i] := loadbitmap(fname);
      if (i > 0) then
         outBmps[i-1] := bmpDiff(bmps[i-1], bmps[i], doGrey, doDesat, doPost, pval);
   end;
   freebitmaps(bmps);
   getbitmapsize(outBmps[0], w, h);
   
   writeln('Processing took ', t.getTime());
   
   for i := 0 to 398 do
   begin
      DrawBitmapDebugImg(outBmps[i]);
      displaydebugimgwindow(w,h);
      freebitmap(outBmps[i]);
      sleep(100);
   end;
end.

Cool!
I just wrote something similar to that right now (very dirty and quick), just for the hack of it.
Once written, I found a video on YouTube (https://www.youtube.com/watch?v=26F_EcacVPU) which I found as a suiting target for a quick test-run:
Result (it will do some processing first):

It's lacking a lot of proper processing and filtering, but as most of the computation was done in Lape I had to limit my self to the bare minimum, else it would be somewhat slow.
- First it removes all moving foreground targets, to get an image of the background.
- Then it simply checks for changing pixels according to the background (with some tolerance), removes the biggest groups (where to many people bundle together).
- It also scales the image to ½ the size so we can get some "real-time tracing" (without having to use a plugin), but that ofcourse limits the quality in this case where the objects we trace are so small.

From: slacky
Cool!
I just wrote something similar to that right now (very dirty and quick), just for the hack of it.
Once written, I found a video on YouTube (https://www.youtube.com/watch?v=26F_EcacVPU) which I found as a suiting target for a quick test-run:
Result (it will do some processing first):

That is really cool, I've wondered how well something like this could be done with Simba.

From: BMWxi
That is really cool, I've wondered how well something like this could be done with Simba.

Edit: I miss-interpreted! :)
Yeah, it works decently, but I would rather have done the main-processing in a plugin, that way I could do much more pre-processing, and real-time processing and still come out faster then Lape, and avoid having to sample the images to ½ to keep the speed going.

Edit: I was running it at about 16 FPS (frames per sec). While the code is by no means optimized.

Thats awesome :O i didnt understand though, this could be used in someway on RS to make it real time and not get any missclicks anymore?

From: undorak7
Thats awesome :O i didnt understand though, this could be used in someway on RS to make it real time and not get any missclicks anymore?

Not the posted code, no. Thats just for messing around to see what things do.

It does look like things run fast enough you could do some amount of live processing.

I don't know how much use it will have outside of a fighter but it's certainly clued me in on the idea that we may be better off searching for things on processed images than on the true screen.

We're basically looking at the same thing as pixelshift, calculating the change in an area over time. But instead of just the count you're looking at the details.

I know the recent thread re: tanning was talking about the color changes in the inventory when it tanned one. This sort of pre/post comparison would definitely highlight that sort of thing.

From: slacky

I just wrote something similar to that right now …
- First it removes all moving foreground targets, to get an image of the background.
- Then it simply checks for changing pixels according to the background (with some tolerance), removes the biggest groups (where to many people bundle together).
- It also scales the image to 1/2 the size so we can get some "real-time tracing" (without having to use a plugin), but that ofcourse limits the quality in this case where the objects we trace are so small.

Haha, you are so awesome. You and Olly are the image masters.

I want to make sure I understand the first part, that was something I was starting to read about.

Something like this?

Take two snaps A and B. Diff them to get the motion. Then take the TPA of the motion and grab those pixels from the OLD image A to put on the background?

From: bonsai
Haha, you are so awesome. You and Olly are the image masters.

I want to make sure I understand the first part, that was something I was starting to read about.

Something like this?

Take two snaps A and B. Diff them to get the motion. Then take the TPA of the motion and grab those pixels from the OLD image A to put on the background?

Pshh! I steal slackys work!

From: bonsai
…
Take two snaps A and B. Diff them to get the motion. Then take the TPA of the motion and grab those pixels from the OLD image A to put on the background?

Something like that. There are many ways you can extract the background, but all works in a similar manner to that.
Basicly what I did for pre-processing/removal of the foreground was:

-> Grab n pictures of the screen with a good interval between am.
-> Initialize a TIA of n size
-> Initialize a Result-image (of target size)
-> For each pixel (x,y) in result do
—> for each image (0 to n-1) add the color of the current pixel (x,y) to the TIA
—> Get the Mode of the TIA (the most common color in it) (with some tolerance)
—> Set Result.Pixel(x,y) := Mode of TIA
—> Reset TIA
-> end

I also used some gaussian blur to reduce pixel-noise.
Now you might wanna split up the process in to a few more steps so you don't have to store 50+ images at a time in memory :)

Tho, this is deff. not the best way to go about it in real footage, as it could get pretty noisy (due to camera-noise, light reflections, etc), not that "prettyness" of the background image is rly important for motion-detection, lol.

From: bonsai
It does look like things run fast enough you could do some amount of live processing.

The gif posted above is not really that close to the real processing speed, it's a gif, it would be much larger (in size) if I did not crop the FPS of it.

From: slacky
The gif posted above is not really that close to the real processing speed, it's a gif, it would be much larger (in size) if I did not crop the FPS of it.

Yes but for runescape needs it's not too bad.

These do a half decent job without any fancy work. Play the script while logged in. Disable smart, move around, etc.

program desatRunescape;
{$DEFINE SMART}
{$include_once srl-6/srl.simba}
var
   bmp, w, h: integer;

procedure desatBmp(var bmp: integer);
var b, g: integer;
begin
   b := copybitmap(bmp);
   g := desaturate(b);
   freebitmaps([bmp, b]);
   bmp := g;
end;
   
function getMotionBitmap(): integer;
var
   image1, image2, work1, work2, diffBmp, i, j, c: integer;     
begin
   image1 := bitmapfromclient(0, 0, w-1, h-1);
   sleep(75);
   image2 := bitmapfromclient(0, 0, w-1, h-1);

   diffBmp := createbitmap(w,h);
   work1 := copybitmap(image1);
   work2 := copybitmap(image2);
   desatBmp(work1);
   desatBmp(work2);

   for i := 0 to w-1 do
      for j := 0 to h-1 do
      begin
         c := fastgetpixel(work2, i, j);
         if (fastgetpixel(work1, i, j) <> c) then
            fastsetpixel(diffBmp, i, j, c);
      end;
      
   result := diffBmp;
   freebitmaps([image1, image2, work1, work2]);
end;

begin
   smartPlugins := ['d3d9.dll'];
   smartEnableDrawing := true;
   SetupSRL();
   getclientdimensions(w, h);
   while (true) do
   begin
      bmp := getMotionBitmap();
      smartImage.drawbitmap(bmp, point(0, 0));
      freebitmap(bmp);
   end;
end.

program greyMotionRunescape;
{$DEFINE SMART}
{$include_once srl-6/srl.simba}
var
   bmp, w, h: integer;

procedure greyBmp(var bmp: integer);
var b, g: integer;
begin
   b := copybitmap(bmp);
   g := greyscalebitmap(b);
   freebitmaps([bmp, b]);
   bmp := g;
end;
   
function getMotionBitmap(): integer;
var
   image1, image2, work1, work2, diffBmp, i, j, c: integer;     
begin
   image1 := bitmapfromclient(0, 0, w-1, h-1);
   sleep(75);
   image2 := bitmapfromclient(0, 0, w-1, h-1);

   diffBmp := createbitmap(w,h);
   work1 := copybitmap(image1);
   work2 := copybitmap(image2);
   greyBmp(work1);
   greyBmp(work2);

   for i := 0 to w-1 do
      for j := 0 to h-1 do
      begin
         c := fastgetpixel(work2, i, j);
         if (fastgetpixel(work1, i, j) <> c) then
            fastsetpixel(diffBmp, i, j, c);
      end;
      
   result := diffBmp;
   freebitmaps([image1, image2, work1, work2]);
end;

begin
   smartPlugins := ['d3d9.dll'];
   smartEnableDrawing := true;
   SetupSRL();
   getclientdimensions(w, h);
   while (true) do
   begin
      bmp := getMotionBitmap();
      smartImage.drawbitmap(bmp, point(0, 0));
      freebitmap(bmp);
   end;
end.

From: bonsai
Yes but for runescape needs it's not too bad.

These do a half decent job without any fancy work. Play the script while logged in. Disable smart, move around, etc.

This is super interesting, I'm gonna run those scripts later tonight when I get home!

Try this one lol. All we need is some slacky magic.

program greyMotionRunescape;
{$DEFINE SMART}
{$include_once srl-6/srl.simba}
var
   bmp: integer;
   captureBox: TBox;

procedure greyBmp(var bmp: integer);
var b, g: integer;
begin
   b := copybitmap(bmp);
   g := greyscalebitmap(b);
   freebitmaps([bmp, b]);
   bmp := g;
end;
   
function getMotionBitmap(b: TBox; n: integer = 1; c: integer = 255): integer;
var
   image1, image2, work1, work2, 
   diffBmp, i, j, w, h, old, new, diffCount: integer;     
begin
   image1 := bitmapfromclient(b);
   sleep(75);
   image2 := bitmapfromclient(b);

   getbitmapsize(image1, w, h);
   diffBmp := createbitmap(w, h);
   work1 := copybitmap(image1);
   work2 := copybitmap(image2);
   greyBmp(work1);
   greyBmp(work2);

   for i := 0 to w-1 with n do
      for j := 0 to h-1 with n do
      begin
         old := fastgetpixel(work1, i, j);
         new := fastgetpixel(work2, i, j);
         if (old <> new) then
         begin
            inc(diffCount);
            if (diffCount > 6000) then break(2);
            if (c = 0) then
               fastsetpixel(diffBmp, i, j, old)
            else if (c = -1) then
               fastsetpixel(diffBmp, i, j, new)
            else
               fastsetpixel(diffBmp, i, j, c);
         end;
      end;
      
   result := diffBmp;
   freebitmaps([image1, image2, work1, work2]);
end;

function hilight(b: integer): TBoxArray;
var
   oldTarget, w, h, i: integer;
   tpa: TPointArray;
   atpa: T2DPointArray;
begin
   oldTarget := GetImageTarget();
   SetTargetBitmap(b);
   getbitmapsize(b, w, h);
   findcolors(tpa, 255, 0, 0, w-1, h-1);
   SetImageTarget(oldTarget);
   
   atpa := tpa.split(5);
   atpa.filterbetween(0, 10);
   atpa.filterbetween(2000,maxint);
 
   smartImage.cleararea(captureBox); // sloppy but cheap
//   smartImage.debugtpa(tpa, true);
   for i := 0 to high(atpa) do
      smartimage.drawbox(atpa[i].getBounds(), clLime);
end;

begin
   smartPlugins := ['d3d9.dll'];
   smartEnableDrawing := true;
   SetupSRL();
   captureBox := mainscreen.getbounds();
   while (true) do
   begin
      bmp := getMotionBitmap(captureBox, 2);
      hilight(bmp);
      freebitmap(bmp);
   end;
end.

From: bonsai
Try this one lol. All we need is some slacky magic.

program greyMotionRunescape;
{$DEFINE SMART}
{$include_once srl-6/srl.simba}
var
   bmp: integer;
   captureBox: TBox;

procedure greyBmp(var bmp: integer);
var b, g: integer;
begin
   b := copybitmap(bmp);
   g := greyscalebitmap(b);
   freebitmaps([bmp, b]);
   bmp := g;
end;
   
function getMotionBitmap(b: TBox; n: integer = 1; c: integer = 255): integer;
var
   image1, image2, work1, work2, 
   diffBmp, i, j, w, h, old, new, diffCount: integer;     
begin
   image1 := bitmapfromclient(b);
   sleep(75);
   image2 := bitmapfromclient(b);

   getbitmapsize(image1, w, h);
   diffBmp := createbitmap(w, h);
   work1 := copybitmap(image1);
   work2 := copybitmap(image2);
   greyBmp(work1);
   greyBmp(work2);

   for i := 0 to w-1 with n do
      for j := 0 to h-1 with n do
      begin
         old := fastgetpixel(work1, i, j);
         new := fastgetpixel(work2, i, j);
         if (old <> new) then
         begin
            inc(diffCount);
            if (diffCount > 6000) then break(2);
            if (c = 0) then
               fastsetpixel(diffBmp, i, j, old)
            else if (c = -1) then
               fastsetpixel(diffBmp, i, j, new)
            else
               fastsetpixel(diffBmp, i, j, c);
         end;
      end;
      
   result := diffBmp;
   freebitmaps([image1, image2, work1, work2]);
end;

function hilight(b: integer): TBoxArray;
var
   oldTarget, w, h, i: integer;
   tpa: TPointArray;
   atpa: T2DPointArray;
begin
   oldTarget := GetImageTarget();
   SetTargetBitmap(b);
   getbitmapsize(b, w, h);
   findcolors(tpa, 255, 0, 0, w-1, h-1);
   SetImageTarget(oldTarget);
   
   atpa := tpa.split(5);
   atpa.filterbetween(0, 10);
   atpa.filterbetween(2000,maxint);
 
   smartImage.cleararea(captureBox); // sloppy but cheap
//   smartImage.debugtpa(tpa, true);
   for i := 0 to high(atpa) do
      smartimage.drawbox(atpa[i].getBounds(), clLime);
end;

begin
   smartPlugins := ['d3d9.dll'];
   smartEnableDrawing := true;
   SetupSRL();
   captureBox := mainscreen.getbounds();
   while (true) do
   begin
      bmp := getMotionBitmap(captureBox, 2);
      hilight(bmp);
      freebitmap(bmp);
   end;
end.

You're gonna wanna tone it down a wee bit, catching boxes on some idle animations.

From: NKN
You're gonna wanna tone it down a wee bit, catching boxes on some idle animations.

They are changing so they highlight. There's a million ways to play with it.

Look at my interpretation of making a background vs. slacky's reponse earlier :spongebob:

I have a video to post in a few but I need to take a call first…

Here's a youtube for you young punks who smoke too much dope like Wu-Tang or people like Flight who don't play RS3.

Or poeple like Robert who fit both categoreis…

[video=youtube;w1twXCstKmI]https://www.youtube.com/watch?v=w1twXCstKmI[/video]

From: bonsai
Here's a youtube for you young punks who smoke too much dope like Wu-Tang or people like Flight who don't play RS3.

Or poeple like Robert who fit both categoreis…

[video=youtube;w1twXCstKmI]https://www.youtube.com/watch?v=w1twXCstKmI[/video]

so much awesomeness duuude im lovin it

It reminds me something I've made in old days. https://villavu.com/forum/showthread.php?t=81759&p=1011385#post1011385
…Much more tricky and slow and Olly said it was useless:p ( and yes he was right atm). Now, with Lape custom functions are possible without huge loss of speed.

What you've posted seems promising!



This is my version:

[video=youtube;W5FaUq6bjaU]https://www.youtube.com/watch?v=W5FaUq6bjaU[/video]

My old PC has hard time running RS. With lower quality delays are smallers. I suppose ETL lags a bit too.


[video=youtube;8YTrVVmQ3C0]https://www.youtube.com/watch?v=8YTrVVmQ3C0[/video]


This is just raw comparision, optimized to (almost) maximum in Lape, of two bitmaps, with TPA of differ pixels on output. I wanted to see if it can work with reasonable speed without any pre/postprocessing. Although, what takes most of time is an iteration through all pixels to compare them, so IMHO this should be in plugin.

@Edit

and another one, this time with plugin:
(FPS is higher, but slightly droped on cam)

[video=youtube;txpgQDcRBmQ]https://www.youtube.com/watch?v=txpgQDcRBmQ[/video]

I didn't know that stream actually moved :p

From: bg5
..

If you don't want your "drawings" to flicker, draw in memory first then blit the entire thing to the screen at once (double buffering).. Drawing directly to the screen isn't a good idea.

From: Brandon
If you don't want your "drawings" to flicker, draw in memory first then blit the entire thing to the screen at once (double buffering).. Drawing directly to the screen isn't a good idea.

Thanks for advice. I really thought that drawing directly on screen will be faster. The double buffering improved projection.

From: bg5
Thanks for advice. I really thought that drawing directly on screen will be faster. The double buffering improved projection.

Glad it worked out for you. If you ever want to know why, then the below can give a very vague but "okay" explanation I guess..

Good link:
http://www.anandtech.com/show/2794/2


My own explanation:

OpenGL & Direct-X use it. The way it works is this:


Buffer 1 - Backbuffer
Buffer 2 - FrontBuffer/Screen


What they do is they draw on the back-buffer. SWAP the back and front-buffer. This way, the back-buffer is now drawn on the screen and the front buffer can be used for drawing all the models and textures.

SWAP again, and again, and again.. So while the graphics card is drawing the front buffer on screen, all the models are being drawn to a buffer in the background. When it swaps, that's 1 frame.. Every swap is 1 frame..

While the models & textures are being drawn in the background to memory (getting ready to go to the front), the graphics card is displaying the front and so on.. it just keeps swapping and that's why you don't see any flickering or lag or anything.. Higher FPS, etc..


There's also tripple buffering, etc.. but double is usually enough. If memory serves me correct (and I hope it does), OpenGL allows 8-buffers I think!



The reason flickers occur in direct or single-buffer drawing is because while you are drawing to the screen, it's being displayed at the same time and it is out of sync. Instead of drawing the entire thing at once, it draws whatever it has at THAT specific time. "That" refers to the time when the GPU and CPU "sync" so that the CPU can upload the data to the GPU. If the CPU is still uploading, the GPU doesn't stop or care.. it just draws whatever is there.. By the time the CPU is done, the GPU is already trying to draw the next frame and so.. you get that annoying flicker.


If you draw to memory first, then send that to the screen all at once, there is no delay, lag, flicker or anything.. And no, it won't slow down anything. It's actually faster since there's less processing it has to do. It's one call :]

This looks awesome, I might try later to incorporate this into our CCTV surveillance to only start saving the recording when it detects a certain amount of movement such that of a human.

This isn't motion related but it seemed like a good place to drop it.

Took a break from my code cleanup to play runescape through RGB filters. Figured I would share the fun!

program rgbRunescape;
{$DEFINE SMART}
{$include_once srl-6/srl.simba}
var
   orig, w, h, brk1, brk2: integer;
   x, y, col, r, g, b: integer;

procedure quit();
begin
   smartImage.clear();
   FreeBitmap(orig);
end;

begin
   addOnTerminate('quit');
   smartPlugins := ['d3d9.dll'];
   smartEnableDrawing := true;
   SetupSRL();
   GetClientDimensions(w, h);
   brk1 := w div 4;
   brk2 := brk1 * 2;
   SmartSetEnabled(__smartCurrentTarget, false);
   while (true) do
   begin
      orig := BitmapFromClient(0, 0, w-1, h-1);
      for x := 0 to w-1 do
         for y := 0 to h-1 do
         begin
            col := FastGetPixel(orig, x, y);
            ColorToRGB(col, r, g, b);
            case x of
               1..brk1:
                  r := b := 0; {keep green}
               brk1+1..brk2:
                  b := g := 0; {keep red}
               else
                  r := g := 0; {keep blue}
            end;
            FastSetPixel(orig, x, y, RGBToColor(r,g,b));
         end;
      smartImage.drawBitmap(orig, point(0, 0));
      FreeBitmap(orig);
   end;
end.