Source code for multi_animal_tracking

__author__ = 'David Tadres'
__project__ = 'PiVR'

import numpy as np
import tkinter as tk
import os
from matplotlib.figure import Figure
from scipy import ndimage
import matplotlib.backends.backend_tkagg as tkagg
import time
from tkinter import messagebox
from skimage import measure
from skimage.draw import line
from matplotlib.patches import Rectangle
from matplotlib import cm
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
import pandas as pd
import json

# Local Modules
import initialize_image_data

large_plots = False

[docs]class MultiAnimalTracking(): """ The Multi-Animal Tracker allows the identification and tracking of several animals in a video or image series. This tracker depends on user input, specifically: #. The user should identify the region in the frame where the animals are to be expected. This helps reduce mis-identification of structures outside that area as animals. #. The user should optimize the detection by using the 'Treshold (STD from Mean)" slider. When doing background subtraction, the current image is subtracted from the mean image. The treshold defined using this slider defines how many standard deviations (e.g. 5 x Standard Deviation) from the mean value of pixel intensities of the subtracted image the animals are expected. In other words - the clearer your animals stand out (large contrast) the higher the treshold can be set. #. The "Minimum filled area" slider gives the user a handle on the animal size: After background subtraction and applying the threshold (see above) the algorithm goes through all the `"blobs" <https://scikit-image.org/docs/dev/api/skimage.measure.html#skimage.measure.regionprops>`_. To determine whether a given blob counts as an animal it compares the number of fixels and compares it to this Minimum filled area. A blob will only count as an animal if it contains equal or more pixels as defined here. #. The "Maximum filled area" slider gives the user a handle on the animal size by defining the maximum area (in pixels) the animal has (see above). #. The "Major over Minor Axis" slider lets the user select for "elongated" objects. The Major and Minor axis are properties of the `"blob" <https://scikit-image.org/docs/dev/api/skimage.measure.html#skimage.measure.regionprops>`_. For animal that are often round (such as fruit fly larva) it is best to keep this parameter at zero. For animals that are rigid such as adult fruit flies, it can be useful set this slider to a number higher than one. #. The "Max Speed Animal [mm/s]" is used during tracking to define realistic travelled distances between two frames. To calculate this, the script takes the pixel/mm and the framerate as recorded in "experiment_settings.json" into account. For example, if you have a fruit fly larva that moves not faster than 2mm/s and you have recorded a video at 5 frames per second at a distance (camera to animals) translating to 5pixel/mm at your chosen resolution a blob can not move more than (2mm/s*5pixel/mm)/5 frames per second = 2 pixel per frame. .. warning:: This feature can lead to unexpected results. If your trajectories look unexpected, try relaxing this parameter (=put a large number, e.g. 200) #. The "Select Rectangular ROI" is a important feature: it allows the selection of a rectangular area using the mouse in the main window. When looking for animals, only the area inside this area is taken into consideration. #. The main window displays the current frame defined by pulling the slider next to "Start Playing". This can be used to optimize the "Image parameters" described above. To just watch the video you can of course also press the "Start Playing" button. The multi-animal tracking algorithm critically depends on optimal image parameters which means that for optimal results **each frame should contain the expected number of animals**. For example, if you are running an experiment with 5 animals the goal is to adjust the image parameters such that for each frame you will have 5 animals. See :ref:`here <ToolsMultianimalTracking>` on how to best achieve this. To help the user find frames where the number of animals is incorrect, the button "Auto-detect blobs" can be very useful. It detects, in each frame, the number of `"blobs" <https://scikit-image.org/docs/dev/api/skimage.measure.html#skimage.measure.regionprops>`_. that fit the image parameters irrespective of distance travelled. See :func:`MultiAnimalTracking.detect_blobs` for details on what that function is doing exactly. Once the user presses the "Track Animals" button, the :func:`MultiAnimalTracking.ask_user_correct_animal_classification` function is called. This function uses the current frame and applies the user defined image parameters to determine the number of animals used in the experiment. It then shows a popup indicating the blobs identified as animals and ask the user if this is correct. If the user decides to go ahead with tracking, the actual tracking algorithm starts. The principle of this multi-animal tracker is the following: #. User has defined the number of expected animals by choosing a frame (i.e. Frame # 50) where the correct number of animals can be identified. #. A numpy array with the correct space for storing X and Y coordinates for all these animals for each frame is pre-allocated #. In the user defined frame (i.e. Frame # 50), the position of each animal is identified. #. The centroid position for each animal is stored in the pre-allocated array. The order is from identified animal top left to bottom right. I.e. the animal that is top left in the image in i.e. Frame #50 will be in position #1 in the numpy array. #. As the user defined frame does not have to be the first frame, the tracking algorithm can run "backwards", i.e. identifying animals in frame 50, 49, 48... and once it reaches zero it will run forward, in our example 51, 52 ... #. In the next frame (which can also be the previous frame as the analysis can run backwards), the blobs that can be animals are again identified using animal parameters. In our example where the starting frame was 50, the "next" frame to be analyzed is 49. #. The centroids in frame 49 are assigned to the previously identified frame by calculating the distance of each centroid to each of the previously identified centroids. Centroids with the smallest distance are assumed to be from the same animal. #. In many multi-animal experiments, animal can touch each other which makes it impossible for the algorithm to distinguish them. For a frame where 2 (or more) touch each other, only one centroid can be assigned to the touching animals. #. Once the animals do not touch anymore, they can be re-idenfied as single animals. To assign them to their previous trajectory the distance to the previously known position of the animal that was lost before. However, for the time that the animal is missing, no assumptions are made and the data is just missing. """ def __init__(self, data_path, colormap, recording_framerate, organisms_and_heuristics): self.path = data_path os.chdir(self.path) # self.csv_data = csv_data # self.background_image = background_image # self.data_path = data_path self.recording_framerate = recording_framerate self.colormap = colormap self.update_overview_bool = True self.playback_speed_options = ('0.1X', '0.5X', '1X', '2X', '5X', 'Custom') self.playback_speed_variable = tk.StringVar() self.playback_speed_variable.set(self.playback_speed_options[2]) self.playback_speed_value = 1.0 try: with open(('experiment_settings.json'), 'r') as file: experiment_variables = json.load(file) try: self.model_organism = experiment_variables['Model Organism'] self.pixel_per_mm = experiment_variables['Pixel per mm'] except KeyError: print('Model Organism and Pixel per mm not found in ' 'in experimental_settings.json') self.model_organism = 'unknown' self.pixel_per_mm = None # put the proper number in here except FileNotFoundError: self.model_organism = 'unknown' self.pixel_per_mm = None # put the proper number in here self.images, framerate = initialize_image_data.get_self_images() try: # takes forever to calculate - save self.background = np.load('Background.npy') # yes, # I tried to save with imageio, trouble with reading the # tiff! self.smooth_images = np.load('smoothed_images.npy') except (FileNotFoundError, OSError): self.smooth_images = ndimage.filters.gaussian_filter( self.images, sigma=0.5) #np.save('smoothed_images.npy', self.smooth_images) self.background = np.mean(self.smooth_images, axis=2) np.save('Background.npy', self.background) self.images = None # free memory! try: self.timestamps = np.load('timestamps.npy') except FileNotFoundError: self.timestamps = None self.image_number = tk.IntVar() self.image_number.set(0) self.callback_func_variable = 0 self.threshold_number = tk.IntVar() self.threshold_number.set(5) self.minimum_filled_area_number = tk.IntVar() self.minimum_filled_area_number.set(20) self.maximum_filled_area_number = tk.IntVar() self.maximum_filled_area_number.set(200) self.major_over_minor_axis_number = tk.DoubleVar() self.major_over_minor_axis_number.set(1) self.number_of_blobs = [] self.manually_jump_to_frame_number = tk.IntVar() # the first ROI is the whole image self.ROI = [[0,0,self.background.shape[0], self.background.shape[1]]] self.ROI_selected = False self.x_release = None self.y_release = None self.x_press = None self.y_press = None self.cidpress = None self.cidrelease = None self.rect = Rectangle((0, 0), 1, 1, alpha=0.1, fill=False, hatch='/') # tracking data self.centroids = None self.bounding_boxes = None self.number_of_animals = tk.IntVar() self.number_of_animals.set(10) self.thresholded_images = None self.max_speed_number = tk.DoubleVar() try: self.max_speed_number.set( organisms_and_heuristics[self.model_organism] ['max_speed_animal_mm_per_s']) except KeyError: if self.pixel_per_mm is None or self.pixel_per_mm == 0: tk.messagebox.showinfo( 'No max speed found', 'Did not find ' + self.model_organism + ' in\n' 'list_of_available_organisms.json.\n' 'In addition, pixel per mm is not defined\n' 'Could not set max speed automatically. Was set\n' 'to 10px/s - please change as needed') self.max_speed_number.set(10) else: tk.messagebox.showinfo( 'No max speed found','Did not find ' + self.model_organism + ' in\n' 'list_of_available_organisms.json. ' 'Could not set max speed automatically. Was set\n' 'to 2mm/s - please change as needed') self.max_speed_number.set(2) # plotting # colormap self.cmap = None # text artist container self.text_artists = [] # scatterplot_artist container self.scat_artists = [] # ROI artist self.ROI_artist = None self.initial_centroids = [] self.initial_bounding_boxes = [] self.child = tk.Toplevel() self.child.grab_set() self.child.wm_title('Semi-Automatic Multi-Animal Tracking') self.child.protocol("WM_DELETE_WINDOW", self.quit_func) self.child_frame = tk.Frame(self.child) self.child_frame.grid(row=0, column=0) # The overview if large_plots: self.fig_overview = Figure(figsize=(11, 7)) else: self.fig_overview = Figure(figsize=(6, 4)) self.ax_overview = self.fig_overview.add_subplot(111) self.image_of_background = self.ax_overview.imshow( self.background, vmin=0, vmax=255, cmap=self.colormap) # self.image_of_background = self.ax_overview.imshow( # np.mean(self.thresholded_images, axis=2)) # turn off the labels and axis to save space # self.ax_overview.axes.get_xaxis().set_ticks([]) # self.ax_overview.axes.get_yaxis().set_ticks([]) # turn on the grid - a bit buggy when user moves around using # the toolbar # self.ax_overview.grid() self.fig_overview.tight_layout() # bind the plot to the GUI - do it in a new frame due to the # inherent pack method of NavigationToolbar overview_frame = tk.Frame(self.child_frame) overview_frame.grid(row=1, column=1, rowspan=3, columnspan=5) self.update_overview_button = tk.Button( overview_frame, text='Updating Overview', command=self.update_overview_func) self.update_overview_button.pack() self.canvas_overview = tkagg.FigureCanvasTkAgg( self.fig_overview, master=overview_frame) self.canvas_overview.draw() self.canvas_overview_background = \ self.canvas_overview.copy_from_bbox(self.ax_overview.bbox) # Add the toolbar overview_toolbar = tkagg.NavigationToolbar2Tk( self.canvas_overview, overview_frame) overview_toolbar.update() # The next line is necessary to actually show the figure self.canvas_overview.get_tk_widget().pack() # plot to the right of the image - displays how many blobs # are identified in each frame. This should just help the # user to get an idea where to look if there are too many/too # few blobs. Crashed/run out animals will be handled afterwards. if large_plots: self.fig_blob_plot = Figure(figsize=(2,7)) else: self.fig_blob_plot = Figure(figsize=(2,4)) self.ax_blob_plot = self.fig_blob_plot.add_subplot(111) # self.ax_blob_plot.set_ylim(0,self.smooth_images.shape[0]) self.ax_blob_plot.set_xlim(0,self.number_of_animals.get()) # todo - put animal number self.ax_blob_plot.set_ylabel('Frame number') self.ax_blob_plot.set_xlabel('# of animals') # need to plot something to get the object self.blob_plot_background, = self.ax_blob_plot.plot( np.zeros(self.smooth_images.shape[2]), np.arange(0,self.smooth_images.shape[2],1)) self.blob_plot_indicator, = self.ax_blob_plot.plot( [0,int(2*self.number_of_animals.get())], [0,0], color='r',lw=1, alpha=0.8,zorder=0, linestyle=':') #print(int(np.ceil(number_of_animals+number_of_animals*0.1))) self.fig_blob_plot.tight_layout() # bind the plot to the GUI - do it in a new frame due to the # inherent pack method of NaviagationToolbar blob_plot_frame = tk.Frame(self.child_frame) blob_plot_frame.grid(row=1, column=6, rowspan=3) self.canvas_blob_plot = tkagg.FigureCanvasTkAgg( self.fig_blob_plot, master=blob_plot_frame) self.canvas_blob_plot.draw() # The next line is necessary to actually show the figure self.canvas_blob_plot.get_tk_widget().pack() # Button that will call the automated blob detection self.blob_detection_button = tk.Button( blob_plot_frame, text='Auto-detect\nblobs', command=self.detect_blobs) self.blob_detection_button.pack() # track animals self.track_animals_frame = tk.Frame(self.child_frame, relief=tk.RIDGE) self.track_animals_frame.grid(row=5, column=6, rowspan=2) self.number_of_animals_label = tk.Label( self.track_animals_frame, text='# of animals in\ncurrent frame') self.number_of_animals_label.grid(row=0, column=0) self.number_of_animals_label_number = tk.Label( self.track_animals_frame, textvariable=self.number_of_animals) self.number_of_animals_label_number.grid(row=0, column=1) self.number_of_animals_label_number.config( font=("Arial", 20, "bold")) # increase size of font for better readibility #self.number_of_animals_entry = tk.Entry( # self.track_animals_frame, # textvariable=self.number_of_animals) #self.number_of_animals_entry.grid(row=1, column=0) self.track_animals_button = tk.Button( self.track_animals_frame, text = 'Track animals', command=self.ask_user_correct_animal_classification) self.track_animals_button.grid(row=1, column=0, columnspan = 2) # Button to interpolate self.interpolate_button = tk.Button( self.track_animals_frame, text = 'Interpolate centroids', state = tk.DISABLED, command=self.interpolate) self.interpolate_button.grid(row=2, column=0, columnspan = 3) # Optimize parameters for automated detection self.detection_frame = tk.LabelFrame( self.child_frame, text='Image parameters') self.detection_frame.grid(row=0,column=0, rowspan=5) # scale to choose number of STDs from mean to threshold self.threshold_scale = tk.Scale( self.detection_frame, from_=0, to=10, resolution=1, label='Threshold (STDs from Mean)', variable=self.threshold_number, orient='horizontal', len=200, command=self.update_visualization ) self.threshold_scale.grid(row=0, column=0) # print('just initialized threshold_scale') # scale to choose minimum filled area self.minimum_filled_area_scale = tk.Scale( self.detection_frame, from_=0, to=100, resolution=1, label='Minimum Filled Area', variable=self.minimum_filled_area_number, orient='horizontal', len=200, command=self.update_visualization ) self.minimum_filled_area_scale.grid(row=1, column=0) # print('just initialized minimum_filled_area_scale') # scale to choose maximum filled area self.maximum_filled_area_scale = tk.Scale( self.detection_frame, from_=0, to=400, resolution=1, label='Maximum Filled Area', variable=self.maximum_filled_area_number, orient='horizontal', len=200, command=self.update_visualization ) self.maximum_filled_area_scale.grid(row=2, column=0) # print('just initialized maximum_filled_area_scale') # scale to choose length ratio self.major_over_minor_axis_scale = tk.Scale( self.detection_frame, from_=1, to=10, resolution=0.1, label='Major over Minor Axis', variable=self.major_over_minor_axis_number, orient='horizontal', len=200, command=self.update_visualization ) self.major_over_minor_axis_scale.grid(row=3, column=0) if self.pixel_per_mm is None: # Entry to choose max speed of the animal self.max_speed_label = tk.Label(self.detection_frame, text='Max Speed Animal [' 'px/s]') self.max_speed_label.grid(row=4, column=0) else: # Entry to choose max speed of the animal self.max_speed_label = tk.Label(self.detection_frame, text='Max Speed Animal ' '[mm/s]') self.max_speed_label.grid(row=4, column=0) self.max_speed_Entry = tk.Entry( self.detection_frame, textvariable=self.max_speed_number, width = 5) self.max_speed_Entry.grid(row=5, column=0) self.select_roi_button = tk.Button( self.detection_frame, text='Select Rectangular ROI', command=self.draw_rectangle) self.select_roi_button.grid(row=6, column=0) self.select_roi_button['bg'] = 'light grey' self.play_frame = tk.Frame( self.child_frame, relief='groove', borderwidth=2) self.play_frame.grid(row=6, column=1, columnspan=5, sticky='w') # button for play self.play_button = tk.Button( self.play_frame, text='Start Playing', command=self.play_func) self.play_button.grid(row=0, column=0) # scale to choose where to play self.image_number_scale = tk.Scale( self.play_frame, from_=0, to=self.smooth_images.shape[2] - 1, resolution=1, label='Frame shown', variable=self.image_number, orient='horizontal', len=400, command=self.update_visualization) self.image_number_scale.grid(row=0, column=1, columnspan=3) self.image_number_scale.set(self.image_number.get()) # print('just initialized image_number_scale') self.playback_speed_frame = tk.Frame( self.child_frame, relief='groove', borderwidth=2) self.playback_speed_frame.grid( row=7, column=1, columnspan=5, sticky='w') # Information about recording framerate self.recording_framerate_label = tk.Label( self.playback_speed_frame, text='Experiment was recorded with ' + repr( self.recording_framerate) + 'fps and is being played back at') self.recording_framerate_label.grid(row=0, column=0) # menu for speed self.speed_menu = tk.OptionMenu(self.playback_speed_frame, self.playback_speed_variable, *self.playback_speed_options) self.speed_menu.grid(row=0, column=1) # more info about recording framerate self.recording_framerate_label_two = tk.Label( self.playback_speed_frame, text='speed') self.recording_framerate_label_two.grid(row=0, column=2) self.manual_jump_frame = tk.Frame( self.child_frame, relief='groove', borderwidth=2) self.manual_jump_frame.grid(row=8, column=1, sticky='w') # Let user manually enter a frame they want to jump to self.manually_jump_to_frame_label = tk.Label( self.manual_jump_frame, text='Enter a frame you want to jump to') self.manually_jump_to_frame_label.grid(row=0, column=0) self.manually_jump_to_frame_text = tk.Entry( self.manual_jump_frame, textvariable = self.manually_jump_to_frame_number, width=5, ) self.manually_jump_to_frame_text.grid(row=0, column=1) self.manually_jump_to_frame_button = tk.Button( self.manual_jump_frame, text='Jump to frame', command=self.manually_jump_to_frame_func) self.manually_jump_to_frame_button.grid(row=0, column=2) # start by playing the recorded experiment self.child.after(100, self.update_visualization()) self.child.after(100, self.callback_func())
[docs] def quit_func(self): """ In order to quit this window and go back to the main GUI, the user needs to press the 'quit' button and this function will be called. """ if tk.messagebox.askokcancel("Quit", "Do you want quit?"): # set main window active again self.child.grab_release() # close the child window self.child.after(0, self.child.destroy())
[docs] def draw_rectangle(self): """ When the user presses the "Select rectangle" Button, this function is called. It connects the mouse button press and release events. Call :func:`MultiAnimalTracking.on_press` and :func:`MultiAnimalTracking.on_release` """ self.ROI_selected = True self.select_roi_button['bg'] = 'red' self.cidpress = self.ax_overview.figure.canvas.mpl_connect( 'button_press_event', self.on_press) self.cidrelease = self.ax_overview.figure.canvas.mpl_connect( 'button_release_event', self.on_release)
[docs] def on_press(self, event): """ Saves x and y position when user presses mouse button on main window """ self.x_press = int(round(event.xdata)) self.y_press = int(round(event.ydata))
[docs] def on_release(self, event): """ Saves x and y position when user releases mouse button on main window. Also takes care of updating the main window with the new ROI """ self.x_release = int(round(event.xdata)) self.y_release = int(round(event.ydata)) if self.y_press < self.y_release: row_min = self.y_press row_max = self.y_release else: row_min = self.y_release row_max = self.y_press if self.x_press < self.x_release: column_min = self.x_press column_max = self.x_release else: column_max = self.x_press column_min = self.x_release self.ROI.append([row_min, column_min, row_max, column_max]) print(self.ROI) self.rect.set_width(column_max - column_min) self.rect.set_height(row_max - row_min) self.rect.set_xy((column_min, row_min)) self.update_visualization() self.ax_overview.figure.canvas.mpl_disconnect(self.cidpress) self.ax_overview.figure.canvas.mpl_disconnect(self.cidrelease) self.select_roi_button['bg'] = 'light grey'
[docs] def update_visualization(self, scale_input=None): """ Updates the embedded matplotlib plots by setting the data to the current image_number """ print('called update visualization') # reset number of animals self.number_of_animals.set(0) if self.update_overview_bool: # print('updating image') # subtract image by converting both the background and # the smoothed images to int16 (from uint8) subtracted_image = self.background.astype(np.int16) - \ self.smooth_images[:,:,self.image_number.get()].astype(np.int16) # calculate the mean and STD mean_image = np.nanmean(subtracted_image) std_image = np.std(subtracted_image) # threshold as defined by the slider thresholded_image = subtracted_image[:,:] \ > mean_image + self.threshold_number.get() \ * std_image # identify all the blobs blobs = measure.regionprops(measure.label(thresholded_image)) self.image_to_plot = \ self.smooth_images[:,:,self.image_number.get()].copy() #hard # copy necessary to update overview for i in range(len(blobs)): try: #print('blob # ' + repr(i) + ' filled area: ' # + repr(blobs[i].filled_area) ) if self.minimum_filled_area_number.get() \ < blobs[i].filled_area \ < self.maximum_filled_area_number.get() \ and \ blobs[i].major_axis_length\ /blobs[i].minor_axis_length \ > self.major_over_minor_axis_number.get() \ and blobs[i].bbox[0] > self.ROI[-1][0]\ and blobs[i].bbox[1] > self.ROI[-1][1] \ and blobs[i].bbox[2] < self.ROI[-1][2] \ and blobs[i].bbox[3] < self.ROI[-1][3]: # the next line takes the bounding box for # each found blob (with constraints defined # above) and compares it to the detected blob # and sets the value for that blob apart #self.image_to_plot[blobs[i].bbox[0]:blobs[i].bbox[2], blobs[i].bbox[1]:blobs[i].bbox[3]][ # blobs[i].filled_image] = 0 # Find it hard to see well when two animals # are counted as one - bounding boxes should # help here try: rr, cc = line(int(blobs[i].bbox[0]), int(blobs[i].bbox[1]), int(blobs[i].bbox[0]), int(blobs[i].bbox[3])) # top horizontal self.image_to_plot[rr, cc] = 0 rr, cc = line(int(blobs[i].bbox[0]), int(blobs[i].bbox[3]), int(blobs[i].bbox[2]), int(blobs[i].bbox[3])) # right vertical self.image_to_plot[rr, cc] = 0 rr, cc = line(int(blobs[i].bbox[2]), int(blobs[i].bbox[1]), int(blobs[i].bbox[2]), int(blobs[i].bbox[3])) # bottom horizontal self.image_to_plot[rr, cc] = 0 rr, cc = line(int(blobs[i].bbox[0]), int(blobs[i].bbox[1]), int(blobs[i].bbox[2]), int(blobs[i].bbox[1])) # left vertical self.image_to_plot[rr, cc] = 0 # update the number of animals self.number_of_animals.set( self.number_of_animals.get() + 1) except IndexError: print('while drawing the bounding box had ' 'index error, likely action at the ' 'edge of the arena') except ZeroDivisionError: # seems to happend during initalization pass self.image_of_background.set_data(self.image_to_plot) self.canvas_overview.restore_region(self.canvas_overview_background) self.ax_overview.draw_artist(self.image_of_background) # after animals have been identified let user scroll # through the experiment. for i in range(len(self.text_artists)): self.text_artists[i].remove() self.text_artists = [] i_text_counter = 0 if self.centroids is not None: for i_text in range(self.centroids.shape[1]): if not np.isnan(self.centroids[0, i_text, self.image_number.get()]): self.text_artists.append( self.ax_overview.text(int( self.bounding_boxes[3, i_text,self.image_number.get()]), int(self.bounding_boxes[0,i_text, self.image_number.get()]), repr(i_text))) try: self.ax_overview.draw_artist( self.text_artists[i_text_counter]) # error happens if user zooms into an area # where no text needs to be updated except IndexError: pass # only add to text_counter if not nan i_text_counter += 1 # also update the rectangle self.ax_overview.add_patch(self.rect) self.rect.set_width(self.ROI[-1][3] - self.ROI[-1][1]) self.rect.set_height(self.ROI[-1][2] - self.ROI[-1][0]) self.rect.set_xy((self.ROI[-1][1], self.ROI[-1][0])) self.ax_overview.draw_artist(self.rect) try: self.scat_artists.remove() self.scat_artists = [] except TypeError: pass self.canvas_overview.blit(self.ax_overview.bbox) print('frame # ' + repr(self.image_number.get())) # also update the blobplot self.blob_plot_indicator.set_ydata([self.image_number.get(), self.image_number.get()]) self.canvas_blob_plot.draw() self.child.update()
[docs] def play_func(self): """ Function is called when user presses the "Start playing" button. """ if self.play_button['text'] == 'Stop Playing': self.play_button['text'] = 'Start Playing' else: self.play_button['text'] = 'Stop Playing' while self.play_button['text'] == 'Stop Playing': if self.image_number.get() < self.smooth_images.shape[2]-1: time_start = time.time() self.update_visualization() self.image_number.set(self.image_number.get() + 1) # play as fast as requested if (time.time() - time_start) * 1000 \ < 1000 / self.recording_framerate \ * (1 / self.playback_speed_value): print(repr(int( round(((1000 / self.recording_framerate * (1 / self.playback_speed_value)) - (time.time() - time_start)))))) self.child.after( int(round(((1000 / self.recording_framerate * (1 / self.playback_speed_value)) - (time.time() - time_start))))) else: print(time.time() - time_start) # notify the user somehow else: self.image_number.set(0)
def callback_func(self): self.child.after(500, self.callback_func) if self.playback_speed_variable.get() != 'Custom': if self.playback_speed_value \ != float(self.playback_speed_variable.get()[:-1]): self.playback_speed_value \ = float(self.playback_speed_variable.get()[:-1]) else: print('Todo: open popup asking user for a custom speed') # Todo open popup asking user for a custom speed
[docs] def manually_jump_to_frame_func(self): """ Function is called when user presses the "Jump to frame" button. """ try: if 0 < int(self.manually_jump_to_frame_number.get()) \ < self.smooth_images.shape[2]: self.image_number.set(int(self.manually_jump_to_frame_number.get())) self.update_visualization() else: messagebox.showerror( "Invalid Input", "You have entered a value smaller than 0 or " "\nlarger than the exisiting number of frames (" + repr( self.smooth_images.shape[2] - 1) +") \n \n Please enter a number between 0 and " + repr(self.smooth_images.shape[2])) except ValueError: messagebox.showerror( "Invalid Input", "You have not entered an Integer number. " "\n \n Please enter a number between 0 and " + repr(self.smooth_images.shape[2] - 1))
[docs] def detect_blobs(self): """ This function is intended to be used "pre-tracking": If the user thinks the Image parameters are ok and they press "Detect blobs" this function is called. It checks for the number of blobs fitting the Image parameters for each frame. This will make it obvious where the image parameters are producing incorrect results. The function does the following: #. Subtract all images from the background image. #. Threshold (binarize) the subtracted image using the user defined Threshold Image parameter. #. Loop through the subtracted frames and call the `"regionprops" <https://scikit-image.org/docs/dev/api/skimage.measure.html#skimage.measure.regionprops>`_. function on each frame. #. Loop through each of the blobs and determine if they are counting as animals, i.e. by comparing their filled area to the user defined minimum and maximum filled area. #. If they count as animals, just count how many per frame do count. #. Plot the blobs identified as animals in the plot on the right side of the main window. """ # reset list self.number_of_blobs = [] subtracted_images = self.background[:,:,np.newaxis].astype(np.int16) \ - self.smooth_images.astype(np.int16) mean_images = np.nanmean(subtracted_images, axis=(0, 1)) std_images = np.std(subtracted_images, axis=(0, 1)) thresholded_images = subtracted_images[:, :] > mean_images \ + self.threshold_number.get() * std_images for i_images in range(self.smooth_images.shape[2]): #print('image number ' + repr(i_images)) blob_counter = 0 blobs = measure.regionprops(measure.label( thresholded_images[self.ROI[-1][0]:self.ROI[-1][2], self.ROI[-1][1]:self.ROI[-1][3],i_images])) for i_blobs in blobs: if self.minimum_filled_area_number.get() \ < i_blobs.filled_area \ < self.maximum_filled_area_number.get(): try: if i_blobs.major_axis_length/i_blobs.minor_axis_length \ > self.major_over_minor_axis_number.get(): blob_counter +=1 except ZeroDivisionError: #small blobs will do this pass self.number_of_blobs.append(blob_counter) # update the plot self.blob_plot_background.set_xdata(self.number_of_blobs) #self.ax_blob_plot.set_ylim(0,len(self.number_of_blobs)) # re-scale the x_lim to - and + 10% of the minimum and maximum number of blobs, respectively self.ax_blob_plot.set_xlim( int(np.floor( min(self.number_of_blobs)-min(self.number_of_blobs)*0.1)), int(np.ceil( max(self.number_of_blobs)+max(self.number_of_blobs)*0.1))) self.canvas_blob_plot.draw()
[docs] def update_overview_func(self): """ Function is called when user presses the "Update Overview Button. Just changes the bool used in :func:`update_visualization` """ if self.update_overview_button['text'] == 'Not updating Overview': self.update_overview_button['text'] = 'Updating Overview' self.update_overview_bool = True if self.play_button['text'] == 'Start Playing': self.update_visualization() else: self.update_overview_button['text'] = 'Not updating Overview' self.update_overview_bool = False
[docs] def ask_user_correct_animal_classification(self): """ This function is called after the user presses "Track Animals". #. Creates a popup window to show the current frame #. Subtracts the current image from the background image #. Thresholds (binarizes) the subtracted image with the user defined Treshold. #. Identifies all blobs in the current image by calling the `"regionprops" <https://scikit-image.org/docs/dev/api/skimage.measure.html#skimage.measure.regionprops>`_. function. #. For each identified blob, determine whether it counts as an animal according to the user defined image parameters. #. If yes, draw a box around that blob. #. Display the resulting image and ask the user if the identified and numbered blobs are indeed animals and if the tracking algorithm should start. .. important:: The number of animals identified here is used as the 'ground truth' of how many animals are present during the experiment. """ second_child = tk.Toplevel() second_child.grab_set() second_child.wm_title('Identify animals') second_child_frame = tk.Frame(second_child) second_child_frame.grid(row=0, column=0) if large_plots: fig_animal_classification = Figure(figsize=(11, 7)) else: fig_animal_classification = Figure(figsize=(6,4)) ax_animal_classification = fig_animal_classification.add_subplot(111) image_of_animal_classification = ax_animal_classification.imshow( self.smooth_images[:,:,self.image_number.get()], vmin=0, vmax=255, cmap=self.colormap) fig_animal_classification.tight_layout() frame_animal_classification = tk.Frame(second_child_frame) frame_animal_classification.grid(row=0, column=0,columnspan=2) canvas_animal_classification = tkagg.FigureCanvasTkAgg( fig_animal_classification, master=frame_animal_classification) canvas_animal_classification.draw() canvas_animal_classification_background = \ canvas_animal_classification.copy_from_bbox( ax_animal_classification.bbox) # Add the toolbar overview_toolbar = tkagg.NavigationToolbar2Tk( canvas_animal_classification, frame_animal_classification) overview_toolbar.update() # The next line is necessary to actually show the figure canvas_animal_classification.get_tk_widget().pack() def cancel_tracking(): # set main window active again second_child.grab_release() # close the child window second_child.after(0, second_child.destroy()) # Button to return to previous window and select a different # frame/different settings cancel_tracking_button = tk.Button(second_child_frame, text='Not good, take me back', command = cancel_tracking) cancel_tracking_button.grid(row=2,column=0) def start_tracking(): # calculate the thresholded images with the STD at this point subtracted_images = self.background[:,:,np.newaxis].astype(np.int16) \ - self.smooth_images.astype(np.int16) mean_images = np.nanmean(subtracted_images,axis=(0,1)) # Have a memory problem: instead of taking the std of all # images, just take the std of a single image in the # middle of the video # TODO: Revisit std_images = np.nanstd( subtracted_images[:,:,-int(subtracted_images.shape[2]/2)]) # TODO: Fix Typo self.thresholed_images = subtracted_images > mean_images \ + self.threshold_number.get()*std_images mean_images = None # free memory std_images = None # free memory # set main window active again second_child.grab_release() # close the child window second_child.after(0, second_child.destroy()) # start tracking self.tracking_start() start_tracking_button = tk.Button( second_child_frame, text='Looks good, start tracking', command = start_tracking) start_tracking_button.grid(row=2,column=1) # TODO: Check what's going on here - This was just done 10 # lines up! subtracted_images = self.background[:,:].astype(np.int16) \ - self.smooth_images[:,:,self.image_number.get()].astype(np.int16) mean_images = np.nanmean(subtracted_images, axis=(0, 1)) std_images = np.std(subtracted_images, axis=(0, 1)) # only one image for speed thresholded_images = subtracted_images[:, :] > mean_images \ + self.threshold_number.get() * std_images blobs = measure.regionprops(measure.label(thresholded_images)) image_to_plot = self.smooth_images[:,:,self.image_number.get()].copy() # reset in case user clicks on tracking twice self.initial_centroids = [] self.initial_bounding_boxes = [] identified_animals = 0 for i_blobs in blobs: try: long_axis = i_blobs.major_axis_length / \ i_blobs.minor_axis_length except ZeroDivisionError: # this happens when the blob is tiny, so just assume # it to be "round" (both axes are same length and # therefore the ratio is 1) long_axis = 1 if self.minimum_filled_area_number.get() \ < i_blobs.filled_area \ < self.maximum_filled_area_number.get() \ and long_axis > self.major_over_minor_axis_number.get() \ and i_blobs.bbox[0] > self.ROI[-1][0] \ and i_blobs.bbox[1] > self.ROI[-1][1] \ and i_blobs.bbox[2] < self.ROI[-1][2] \ and i_blobs.bbox[3] < self.ROI[-1][3]: try: rr, cc = line(int(i_blobs.bbox[0]), int(i_blobs.bbox[1]), int(i_blobs.bbox[0]), int(i_blobs.bbox[3])) # top horizontal image_to_plot[rr, cc] = 0 rr, cc = line(int(i_blobs.bbox[0]), int(i_blobs.bbox[3]), int(i_blobs.bbox[2]), int(i_blobs.bbox[3])) # right # vertical image_to_plot[rr, cc] = 0 rr, cc = line(int(i_blobs.bbox[2]), int(i_blobs.bbox[1]), int(i_blobs.bbox[2]), int(i_blobs.bbox[3])) # bottom horizontal image_to_plot[rr, cc] = 0 rr, cc = line(int(i_blobs.bbox[0]), int(i_blobs.bbox[1]), int(i_blobs.bbox[2]), int(i_blobs.bbox[1])) # left vertical image_to_plot[rr, cc] = 0 self.initial_centroids.append(i_blobs.centroid) # come as tuples anyway, no need to list self.initial_bounding_boxes.append(i_blobs.bbox) # come as tuples anyway, no need to list identified_animals += 1 except IndexError: print('while drawing the bounding box had index ' 'error, likely action at the edge of the arena') animal_number = tk.Label( second_child_frame, text= repr(identified_animals) + ' animals have been identified ' '\nDuring tracking, this is the expected # of ' 'animals') animal_number.config(font=("Helvetica", 16)) animal_number.grid(row=1, column=0, columnspan=2) image_of_animal_classification.set_data(image_to_plot) canvas_animal_classification.restore_region( canvas_animal_classification_background) ax_animal_classification.draw_artist(image_of_animal_classification) for i_text in range(len(self.initial_centroids)): ax_animal_classification.text( int(self.initial_bounding_boxes[i_text][3]), int(self.initial_bounding_boxes[i_text][0]), repr(i_text)) canvas_animal_classification.draw()
[docs] def tracking_start(self): """ This function organizes the tracking of the animals. It pre-allocates the numpy array for the centroid positions after identifying the correct number of animals in the current frame. The actual tracking function, the tracking_loop(), is defined locally in this function. the tracking_loop() function is called in the correct order in here. If the details in the documentation of this class are not sufficient please have a look at the heavily annotated source code of tracking_loop() function (line 1228) """ print('tracking starts') # Identify all blobs in the image that user has defined as a # good image with the user parameters provided. This is # identical to what happend in func # ask_user_correct_animal_classification() but cheap, # so repeated here to be more explicit (and not moving # variables around) blobs = measure.regionprops(measure.label( self.thresholed_images[:, :, self.image_number.get()])) blob_counter = 0 # Just count how many blobs are defined as animals for i_blobs in blobs: # heuristics to detect blobs that look like animals - try: long_axis = i_blobs.major_axis_length / \ i_blobs.minor_axis_length except ZeroDivisionError: # this happens when the blob is tiny long_axis = 1 if self.minimum_filled_area_number.get() \ < i_blobs.filled_area \ < self.maximum_filled_area_number.get() \ and i_blobs.minor_axis_length > 0 \ and long_axis > self.major_over_minor_axis_number.get() \ and i_blobs.bbox[0] > self.ROI[-1][0] \ and i_blobs.bbox[1] > self.ROI[-1][1] \ and i_blobs.bbox[2] < self.ROI[-1][2] \ and i_blobs.bbox[3] < self.ROI[-1][3]: blob_counter += 1 # As the user just confirmed (Clicked on button that said # start tracking after presenting identified animals) that # the number of blobs corresponds to the number of animals # pre-allocated centroid and bounding boxes self.centroids = np.zeros((3, blob_counter, self.smooth_images.shape[2])) self.centroids.fill(np.nan) self.bounding_boxes = np.zeros((4, blob_counter, self.smooth_images.shape[2])) self.bounding_boxes.fill(np.nan) # an empty array that is taken through the tracking loop. # Saves the position of the animals that are currently # not being detected. missing_animal = np.zeros((2, blob_counter)) missing_animal.fill(np.nan) # identify the inital position of the animal - Necessary as # this will be treated as a 'ground truth' about both the # number of animals that should be detected as well as the # position of the animals. As this tracking algorithm mainly # goes for distance between blobs this is essential to be # able to get semi-correct assignment of animals blob_counter = 0 for i_blobs in blobs: # heuristics to detect blobs that look like animals if self.minimum_filled_area_number.get() \ < i_blobs.filled_area \ < self.maximum_filled_area_number.get() \ and i_blobs.minor_axis_length > 0 \ and i_blobs.major_axis_length \ / i_blobs.minor_axis_length \ > self.major_over_minor_axis_number.get() \ and i_blobs.bbox[0] > self.ROI[-1][0] \ and i_blobs.bbox[1] > self.ROI[-1][1] \ and i_blobs.bbox[2] < self.ROI[-1][2] \ and i_blobs.bbox[3] < self.ROI[-1][3]: # assign centroid position as int (index only can take int) # row, then column, number of animals at this position self.centroids[:, blob_counter, self.image_number.get()] = \ int(round(i_blobs.centroid[0])), \ int(round(i_blobs.centroid[1])), 1 # also assign original bounding box self.bounding_boxes[:, blob_counter, self.image_number.get()] = \ int(i_blobs.bbox[0]),\ int(i_blobs.bbox[1]),\ int(i_blobs.bbox[2]),\ int(i_blobs.bbox[3]) blob_counter += 1 def tracking_loop(backwards = True): """ As this is a post-hoc analysis the analysis can go backwards and forwards in time. The principle of this algorithm is that the user provides a frame where all animals are visible so that location and number can be classified. This function takes the current image (variable bound to image_number_scale: self.image_number), identifies all the blobs that look like animals. It then calculates the distance of the centroids in the current frame to the centroids in the previously analyzed frame. If no animal has been detected in the last frame (i.e. because the animals crashed or because they were hiding in the shadows) just take the last known position and calculate the distance. """ # identify all blobs in image blobs = measure.regionprops(measure.label( self.thresholed_images[:, :, i_frame])) animals_counted = 0 # The following three arrays hold the data that is # necessary to correctly assign each blob to a previously # identified animal. # The easiest way to think about these is to think about # stacking each of them in a excel sheet: minimal_dist # will be in the first column (A) and hold rows 1-x ( # x=number of animals). It will hold (once it has been # filled) the minimal distance (in pixels) between the # blobs in the current frame and the previously analyzed # frame. # animal_index will be in the second column (B) from row # 1-x. It holds the index (== identifier) of the animal # identified in the PREVIOUSLY analyzed frame. # blob_index: As many more blobs are identified (cameras # are noisy, experimental setups not perfect) there are # usually a ton of blobs identified. It can easily range # into double, sometimes triple digits. Using the user # input for minimal and maximal filled area and the major # divided by minor axis most of these blobs can be # discarded as not being animals. The blob_index holds # the index that points in the blob 'array' to the blob # we're interested in. In the example above (excel sheet) # the blob index would be in column C and tell us which # blob needs to be assigned as being which animal # Typical example (AFTER the assignment, of course): # minimal_dist animal_index blob_index # 1.423 0 5 # 2.534 2 3 # 0.436 1 10 minimal_dist = np.zeros((self.centroids.shape[1])) # filled with NaNs as zeros would always be the smallest # distance minimal_dist.fill(np.nan) animal_index = np.zeros((self.centroids.shape[1])) animal_index.fill(np.nan) blob_index = np.zeros((self.centroids.shape[1])) blob_index.fill(np.nan) # again, count number of blobs that count as animals in # the current frame for j_blob in blobs: if self.minimum_filled_area_number.get() \ < j_blob.filled_area \ < self.maximum_filled_area_number.get() \ and j_blob.minor_axis_length > 0 \ and j_blob.major_axis_length / j_blob.minor_axis_length \ > self.major_over_minor_axis_number.get() \ and j_blob.bbox[0] > self.ROI[-1][0] \ and j_blob.bbox[1] > self.ROI[-1][1] \ and j_blob.bbox[2] < self.ROI[-1][2] \ and j_blob.bbox[3] < self.ROI[-1][3]: animals_counted += 1 # creating this working copy of the previously analyzed # self.centroids makes the code much more readable. if backwards: previous_centroid_positions = \ self.centroids[:2, :, i_frame + 1].copy() # if we don't go backwards we are going forward. The last # frame would then be in the past. else: previous_centroid_positions = \ self.centroids[:2, :, i_frame - 1].copy() if backwards: # count number of animals counted in previous frame animals_counted_in_previous_frame = \ np.nansum(self.centroids[2, :, i_frame + 1]) # if we don't go backwards we are going forward. The last # frame would then be in the past. else: animals_counted_in_previous_frame =\ np.nansum(self.centroids[2, :, i_frame - 1]) # check if there are less animals in the currently # analyzed frame compared to the last if animals_counted_in_previous_frame \ < self.centroids.shape[1]: # which index is was missing last frame? for i_missing in range(len( np.where(np.isnan( previous_centroid_positions[0]))[0])): # find the index of the missing animal in the # last frame np.where(np.isnan( previous_centroid_positions[0]))[0][i_missing] # just add a one for each frame that the animal # is missing number_of_frames_animal_lost[np.where(np.isnan( previous_centroid_positions[0]))[0][i_missing]] \ += 1 # If an animal that should exist could not have been # assigned a blob in the previously analyzed frame it # will be indicated as np.nan. As it can appear again ( # either because animals part again after a crash or # because the the animal comes back from hiding) take the # last known position and try to assign any blobs in the # currently analyzed frame to the 'lost' animal. # It's a for loop to make sure it scales easily for m in range(len(np.argwhere(np.isnan( previous_centroid_positions[0, :])))): # assign to previous centroid position search array # so that distance to the current centroid position can # be calculated. The idea really is just to catch # animal that re-appear. Shouldn't lead to a lot of # jumping around of the centroid previous_centroid_positions[:, np.argwhere(np.isnan( previous_centroid_positions[0, :]))[0][0]] \ = missing_animal[:, np.argwhere(np.isnan( previous_centroid_positions[0, :]))[0][0]] # reset the animal counter animals_counted = 0 # go through all the blobs for j_blob in range(len(blobs)): # Use user provided rules to identify blobs that look like animals. if self.minimum_filled_area_number.get() \ < blobs[j_blob].filled_area \ < self.maximum_filled_area_number.get() \ and blobs[j_blob].minor_axis_length \ > 0 and\ blobs[j_blob].major_axis_length \ / blobs[j_blob].minor_axis_length \ > self.major_over_minor_axis_number.get() \ and blobs[j_blob].bbox[0] \ > self.ROI[-1][0] \ and blobs[j_blob].bbox[1] \ > self.ROI[-1][1] \ and blobs[j_blob].bbox[2] \ < self.ROI[-1][2] \ and blobs[j_blob].bbox[3] \ < self.ROI[-1][3]: # switch that gets turned on if animal has been # found - need one for each blob found_animal = False # for each blob that is accepted as an animal, # calculate the distance to all blobs in previous # frame - only keep the minimal distance and the # animal that had the minimal distance. Number of # position also indicates which position for n_animal in range(previous_centroid_positions.shape[1]): # Check if the animal index is already taken. # If so, first come first serve, just skip # that previous animal position! This solves # the problem that when two animals crash it # can happen that when they part again the # minimal distance might be pointing to the # same previously defined animal which leads # to loss of one of the two animals. This if # clause ensures that we never have two # animals assigned to the the same animal # index. It's better to have the NaN and # interpolate afterwards if n_animal not in animal_index: # for readability, explicity calculate distance here: current_dist = np.linalg.norm( previous_centroid_positions[:, n_animal] - np.asarray((blobs[j_blob].centroid[0], blobs[j_blob].centroid[1] )) ) # as the minimal_dist array is filled # with NaNs we can just check if this is # the first time in this loop we are # trying to assign a minimal distance to # this animal if np.isnan(minimal_dist[animals_counted]): # only assign the minimal_dist if the minimal dist is actually realistic if current_dist \ < max_speed_pixel \ * number_of_frames_animal_lost[n_animal]: # calculate eucledian minimal # distance between current blob # centroid and previously # identified centroid of # currently analyzed minimal_dist[animals_counted] = current_dist # previous animal index animal_index[animals_counted] = n_animal # current animal as blob index blob_index[animals_counted] = j_blob # turn switch found_animal = True else: # only assign the minimal_dist if the minimal dist is actually realistic if current_dist < max_speed_pixel * number_of_frames_animal_lost[n_animal]: # if minimal distance larger than distance to next previous animals if current_dist < minimal_dist[animals_counted]: # update the minimal distance to the smaller one minimal_dist[animals_counted] = current_dist # and of course also the index for both the previous animal animal_index[animals_counted] = n_animal # and where to find the current animal in the blob index blob_index[animals_counted] = j_blob if found_animal: animals_counted += 1 # assign each animal to the closest previous animal - # in case an animal is lost/crashed it will just stay empty # as no new animal is closer than to another, still # existing animal. for n_animal_assignment in range(self.centroids.shape[1]): if not np.isnan(blob_index[n_animal_assignment]): self.centroids[:, int(animal_index[n_animal_assignment]), i_frame] = \ int(blobs[int(blob_index[int(n_animal_assignment)])].centroid[0]), \ int(blobs[int(blob_index[int(n_animal_assignment)])].centroid[1]), 1 self.bounding_boxes[:, int(animal_index[n_animal_assignment]), i_frame] = \ int(blobs[int(blob_index[int(n_animal_assignment)])].bbox[0]), \ int(blobs[int(blob_index[int(n_animal_assignment)])].bbox[1]), \ int(blobs[int(blob_index[int(n_animal_assignment)])].bbox[2]), \ int(blobs[int(blob_index[int(n_animal_assignment)])].bbox[3]) # if that animl was lost before, reset counter to 1 as it was just found again! if number_of_frames_animal_lost[int(animal_index[n_animal_assignment])] != 1: number_of_frames_animal_lost[int(animal_index[n_animal_assignment])] = 1 # How many previously identified animals could not be assigned a blob this frame? if np.nansum(self.centroids[2, :, i_frame]): # for each of those: for m in range(len(np.argwhere(np.isnan(self.centroids[2, :, i_frame])))): # only insert the nan in case it hasn't been a # nan before. Essentially we want to save the # last known position of the animal so that we # can interpolate afterwards if np.isnan(missing_animal[0, np.argwhere(np.isnan(self.centroids[2, :, i_frame]))[m][0]]): if backwards: missing_animal[:, np.argwhere(np.isnan(self.centroids[2, :, i_frame]))[m][0]] = \ self.centroids[0:2, np.argwhere(np.isnan(self.centroids[2, :, i_frame]))[m][0], i_frame + 1] else: missing_animal[:, np.argwhere(np.isnan(self.centroids[2, :, i_frame]))[m][0]] = \ self.centroids[0:2, np.argwhere(np.isnan(self.centroids[2, :, i_frame]))[m][0], i_frame - 1] # plotting # Allows user to change this bool even while analyzing - # much faster when not updating! if self.update_overview_bool: # hard copy necessary to update overview self.image_to_plot = self.smooth_images[:,:,i_frame].copy() # for each animal for i_plot in range(self.centroids.shape[1]): # that could be identified if not np.isnan(self.bounding_boxes[0, i_plot, i_frame]): # add a bounding box - try except should only # catch the case where animal is hugging the # edge and bounding box is would be outside # of frame - shouldn't happen try: rr, cc = line(int(self.bounding_boxes[0,i_plot, i_frame]), int(self.bounding_boxes[1,i_plot, i_frame]), int(self.bounding_boxes[0,i_plot, i_frame]), int(self.bounding_boxes[3,i_plot, i_frame])) # top horizontal self.image_to_plot[rr, cc] = 0 rr, cc = line(int(self.bounding_boxes[0,i_plot, i_frame]), int(self.bounding_boxes[3,i_plot, i_frame]), int(self.bounding_boxes[2,i_plot, i_frame]), int(self.bounding_boxes[3,i_plot, i_frame])) # right vertical self.image_to_plot[rr, cc] = 0 rr, cc = line(int(self.bounding_boxes[2,i_plot, i_frame]), int(self.bounding_boxes[1,i_plot, i_frame]), int(self.bounding_boxes[2,i_plot, i_frame]), int(self.bounding_boxes[3,i_plot, i_frame])) # bottom horizontal self.image_to_plot[rr, cc] = 0 rr, cc = line(int(self.bounding_boxes[0,i_plot, i_frame]), int(self.bounding_boxes[1,i_plot, i_frame]), int(self.bounding_boxes[2,i_plot, i_frame]), int(self.bounding_boxes[1,i_plot, i_frame])) # left vertical self.image_to_plot[rr, cc] = 0 except IndexError: print('while drawing the bounding box had' ' index error, likely action at the ' 'edge of the arena') # manual updating of plot to gain speed self.image_of_background.set_data(self.image_to_plot) self.canvas_overview.restore_region(self.canvas_overview_background) self.ax_overview.draw_artist(self.image_of_background) # draw the ROI indicator self.ax_overview.add_patch(self.rect) self.rect.set_width(self.ROI[-1][3] - self.ROI[-1][1]) self.rect.set_height(self.ROI[-1][2] - self.ROI[-1][0]) self.rect.set_xy((self.ROI[-1][1], self.ROI[-1][0])) self.ax_overview.draw_artist(self.rect) # after animals have been identified let user scroll # through the experiment. for i in range(len(self.text_artists)): self.text_artists[i].remove() self.text_artists = [] # label the bounding boxes with the index of the # array, which of course is the identity of the animals # at this point i_text_counter = 0 for i_text in range(self.centroids.shape[1]): # make sure there's no NaN if not np.isnan(self.centroids[0,i_text,i_frame]): self.text_artists.append( self.ax_overview.text(int( self.bounding_boxes[3, i_text,i_frame]), int( self.bounding_boxes[ 0, i_text, i_frame]), repr(i_text))) try: self.ax_overview.draw_artist( self.text_artists[i_text_counter]) # Error happens if user zooms in an area # where this text box does not need to be drawn except IndexError: pass # only increment text_counter if not nan! i_text_counter += 1 self.canvas_overview.blit(self.ax_overview.bbox) # always update frame indicator on the blobplot self.blob_plot_indicator.set_ydata([ self.image_number.get(), self.image_number.get()]) self.canvas_blob_plot.draw() self.child.update() start_frame = self.image_number.get() number_of_frames_animal_lost = np.ones((blob_counter)) if self.pixel_per_mm is None or self.pixel_per_mm == 0: max_speed_pixel = (self.max_speed_number.get() / self.recording_framerate * 2.5) else: max_speed_pixel = (self.max_speed_number.get() * self.pixel_per_mm)/\ self.recording_framerate * 2.5 #max_speed_pixel = 3.8 for i_frame in reversed(range(start_frame)): self.image_number.set(i_frame) tracking_loop(backwards=True) for i_frame in range(start_frame, self.centroids.shape[2]): self.image_number.set(i_frame) tracking_loop(backwards=False) #print(self.centroids[:,:,0:10]) self.interpolate_button.config(state="normal") # plot the overview self.cmap = self.get_cmap(self.centroids.shape[1]+1) fig = Figure(figsize=(10,10)) canvas = FigureCanvas(fig) ax = fig.add_subplot(111) ax.imshow(self.background, cmap='Greys_r') for i in range(self.centroids.shape[1]): ax.scatter(x=self.centroids[1, i, ~np.isnan(self.centroids[1, i, :])], y=self.centroids[0, i, ~np.isnan( self.centroids[1, i, :])], c=np.array(self.cmap(i)).reshape(1,4)) canvas.print_figure('Multi_animal_outpout_wo_interpolation.jpg') csv_object = pd.DataFrame({'Frame' : np.arange(0, self.centroids.shape[2])}) if self.timestamps is not None: # time in timestamp is saved as epoch time - to get time since beginning of experiment just subtract # the first value from every value csv_object['Time'] = self.timestamps[:,1]-self.timestamps[0,1] messagebox.showinfo('Tracking Done', 'Finished Tracking.' '\n\n' 'Find the "XY_position.csv" file in the' '\n experimental folder') for i in range(self.centroids.shape[1]): csv_object['X-Centroid, Animal#' + repr(i)] = self.centroids[1, i, :] csv_object['Y-Centroid, Animal#' + repr(i)] = self.centroids[0, i, :] csv_object.to_csv('XY_postion.csv', sep=',') self.show_tracking_result()
[docs] def interpolate(self): """ During tracking it can happen that animals are not identified in every frame. This function allows to interpolate the trajectories. .. warning:: This is an experimental feature. It can produce very wrong results For each identified animal there is "last frame" where it has been identified and a "new frame" where it is identified again. This function assumes that the animal moved with a constant speed and in linear fashion and just does a linear interpolation between these coordinates. .. important:: An important assumption is that the initial assignment was relatively correct. Small errors can lead to huge effects when using the interpolation function """ csv_object = None csv_object = pd.DataFrame({'Frame' : np.arange(0, self.centroids.shape[2])}) for i in range(self.centroids.shape[1]): csv_object['X-Centroid, Animal#' + repr(i)] = self.centroids[1, i, :] csv_object['Y-Centroid, Animal#' + repr(i)] = self.centroids[0, i, :] try: # Not totally sure if pandas is rounding before converting to int. csv_object.interpolate().astype(int).to_csv('XY_position_interpolated.csv', sep=',') except ValueError: csv_object.interpolate().to_csv('XY_position_interpolated_non_int.csv', sep=',') self.image_to_plot = self.smooth_images[:, :, -1].copy() self.image_of_background.set_data(self.image_to_plot) self.canvas_overview.restore_region(self.canvas_overview_background) self.ax_overview.draw_artist(self.image_of_background) # draw the ROI indicator self.ax_overview.add_patch(self.rect) self.rect.set_width(self.ROI[-1][3] - self.ROI[-1][1]) self.rect.set_height(self.ROI[-1][2] - self.ROI[-1][0]) self.rect.set_xy((self.ROI[-1][1], self.ROI[-1][0])) self.ax_overview.draw_artist(self.rect) # draw the scatterplot indicating the position of the animals try: self.scat_artists.remove() self.scat_artists = [] except TypeError: pass for i in range(self.centroids.shape[1]): try: self.scat_artists.append( self.ax_overview.scatter( x=csv_object['X-Centroid, Animal#' + repr(i)].interpolate().astype(int), y=csv_object['Y-Centroid, Animal#' + repr( i)].interpolate().astype(int), c=np.array(self.cmap(i)).reshape(1,4))) except ValueError: self.scat_artists.append( self.ax_overview.scatter( x = csv_object[pd.notnull( csv_object['X-Centroid, Animal#' +repr(i)])]['X-Centroid, Animal#' + repr(i)], y = csv_object[pd.notnull( csv_object['Y-Centroid, Animal#' +repr(i)])]['Y-Centroid, Animal#' + repr(i)] )) messagebox.showwarning( 'Animal not identified', 'At least one animal has not been assigned centroid' '\npositions at either the first or last couple of frames') self.ax_overview.draw_artist(self.scat_artists[i]) self.canvas_overview.blit(self.ax_overview.bbox) self.child.update()
def show_tracking_result(self): # Call the plot that shows the traces after plotting print('plotting in main window') self.image_to_plot = self.smooth_images[:, :, -1].copy() self.image_of_background.set_data(self.image_to_plot) self.canvas_overview.restore_region(self.canvas_overview_background) self.ax_overview.draw_artist(self.image_of_background) # draw the ROI indicator self.ax_overview.add_patch(self.rect) self.rect.set_width(self.ROI[-1][3] - self.ROI[-1][1]) self.rect.set_height(self.ROI[-1][2] - self.ROI[-1][0]) self.rect.set_xy((self.ROI[-1][1], self.ROI[-1][0])) self.ax_overview.draw_artist(self.rect) # draw the scatterplot indicating the for i in range(self.centroids.shape[1]): scat_artist = self.ax_overview.scatter( x=self.centroids[1, i, ~np.isnan(self.centroids[1, i, :])], y=self.centroids[0, i, ~np.isnan(self.centroids[1, i, :])], c=np.array(self.cmap(i)).reshape(1,4)) self.ax_overview.draw_artist(scat_artist) self.canvas_overview.blit(self.ax_overview.bbox) self.child.update() def get_cmap(self, n, name='hsv'): #Returns a function that maps each index in 0, 1, ..., # n-1 to a distinct RGB color; the keyword argument name must # be a standard mpl colormap name. return cm.get_cmap(name, n)