[PYTHON] Create a video player with PySimpleGUI + OpenCV 3 Addition of mask function
Introduction
In addition to the previous article, we will add a region of interest (ROI) to be selected as the operation target and a process (mask) that targets only a specific part.
Make a video player with PySimpleGUI + OpenCV 2 Add ROI setting and save function (DIVX, MJPG, GIF)
What you can do with this article
You can apply processing to a specific part (mask) in the ROI.
The mask selection converts the image to HSV so that you can select the numerical value with each slider. The mask part is displayed in a separate window so that you can check it.
If you select red, you need to select two ranges of Hue values, 0-20 and 110-255, so create a checkbox called Hue Reverse, and if the checkbox is selected, The value outside the two sliders is set as the mask range.
Hue Reverse [Off]
The range between the left and right sliders of Hue is the mask range.
Hue Reverse [On]
The outside of the left and right sliders of Hue is the mask range. The mask range can be inverted.
You can save a specified range of videos as DIVX, MJEG, GIF.
Read file
Select the file to load using the GUI.
import PySimpleGUI as sg
import cv2
import numpy as np
from PIL import Image
from pathlib import Path
def file_read():
'''
Select a file to read
'''
fp = ""
#GUI layout
layout = [
[
sg.FileBrowse(key="file"),
sg.Text("File"),
sg.InputText()
],
[sg.Submit(key="submit"), sg.Cancel("Exit")]
]
#WINDOW generation
window = sg.Window("File selection", layout)
#Event loop
while True:
event, values = window.read(timeout=100)
if event == 'Exit' or event == sg.WIN_CLOSED:
break
elif event == 'submit':
if values[0] == "":
sg.popup("No file has been entered.")
event = ""
else:
fp = values[0]
break
window.close()
return Path(fp)
Color detection by HSV
HSV conversion → Mask processing is made into a function. Create a mask from the min and max values of H, S, and V received from the GUI, and create a mask image with cv2.bitwize_and ().
def hsv(frame, H_max, H_min, S_max, S_min, V_max, V_min, reverse=False):
frame_hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
if reverse:
lower1 = np.array([0, int(S_min), int(V_min)])
upper1 = np.array([int(H_min), int(S_max), int(V_max)])
mask1 = cv2.inRange(frame_hsv, lower1, upper1)
lower2 = np.array([int(H_max), int(S_min), int(V_min)])
upper2 = np.array([255, int(S_max), int(V_max)])
mask2 = cv2.inRange(frame_hsv, lower2, upper2)
mask = mask1 + mask2
frame = cv2.bitwise_and(frame, frame, mask=mask)
# mask = cv2.bitwise_and(frame, mask, mask=mask)
else:
lower = np.array([int(H_min), int(S_min), int(V_min)])
upper = np.array([int(H_max), int(S_max), int(V_max)])
mask = cv2.inRange(frame_hsv, lower, upper)
frame = cv2.bitwise_and(frame, frame, mask=mask)
return frame
class Main:
def __init__(self):
self.fp = file_read()
self.cap = cv2.VideoCapture(str(self.fp))
#Video save flag
self.rec_flg = False
#Get the first frame
#Check if it can be obtained
self.ret, self.f_frame = self.cap.read()
self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
#If you can get the frame, get various parameters
if self.ret:
self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
#Acquisition of video information
self.fps = self.cap.get(cv2.CAP_PROP_FPS)
self.width = int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH))
self.height = int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
self.total_count = self.cap.get(cv2.CAP_PROP_FRAME_COUNT)
# ROI
self.frames_roi = np.zeros((5, self.height, self.width))
As a mask image, prepare a grayscale mask image of the same size as the ROI.
#Definition of mask image
self.mask = np.zeros_like(self.f_frame[:, :, 0])
#Save original size
self.org_width = self.width
self.org_height = self.height
#Frame related
self.frame_count = 0
self.s_frame = 0
self.e_frame = self.total_count
#Image cutout position
self.x1 = 0
self.y1 = 0
self.x2 = self.width
self.y2 = self.height
#Playback pause flag
self.stop_flg = False
#Control of mouse movement
#Whether the mouse button is pressed
self.mouse_flg = False
self.event = ""
#Whether to apply operations to ROI
self.roi_flg = True
cv2.namedWindow("Movie")
#Mouse event callback registration
cv2.setMouseCallback("Movie", self.onMouse)
#Exit if the frame could not be obtained
else:
sg.Popup("Failed to read the file.")
return
#Mouse event
def onMouse(self, event, x, y, flags, param):
#Left click
if event == cv2.EVENT_LBUTTONDOWN:
self.x1 = self.x2 = x
self.y1 = self.y2 = y
#Start drawing a rectangle. Press the mouse once to start drawing a rectangle.
self.mouse_flg = True
#Pause the calculation of the ROI part
self.roi_flg = False
return
elif event == cv2.EVENT_LBUTTONUP:
#Stop updating rectangles
self.mouse_flg = False
#Start calculation on ROI
self.roi_flg = True
#If the ROI selection is 0, reset it and stop the ROI calculation.
if (
x == self.x1
or y == self.y1
or x <= 0
or y <= 0
):
self.x1 = 0
self.y1 = 0
self.x2 = self.width
self.y2 = self.height
return
# x1 <to be x2
elif self.x1 < x:
self.x2 = x
else:
self.x2 = self.x1
self.x1 = x
if self.y1 < y:
self.y2 = y
else:
self.y2 = self.y1
self.y1 = y
#Show ROI range
print(
"ROI x:{0}:{1} y:{2}:{3}".format(
str(self.x1),
str(self.x2),
str(self.y1),
str(self.y2)
)
)
return
#Continues to display rectangle when mouse is pressed
if self.mouse_flg:
self.x2 = x
self.y2 = y
return
def run(self):
# GUI #######################################################
#GUI layout
#Tab 1
T1 = sg.Tab("Basic", [
[
sg.Text("Resize ", size=(13, 1)),
sg.Slider(
(0.1, 4),
1,
0.01,
orientation='h',
size=(40, 15),
key='-RESIZE SLIDER-',
enable_events=True
)
],
[
sg.Checkbox(
'blur',
size=(10, 1),
key='-BLUR-',
enable_events=True
),
sg.Slider(
(1, 10),
1,
1,
orientation='h',
size=(40, 15),
key='-BLUR SLIDER-',
enable_events=True
)
],
])
T2 = sg.Tab("processing", [
[
sg.Checkbox(
'gray',
size=(10, 1),
key='-GRAY-',
enable_events=True
)
],
])
GUI for mask setting
Prepare a tab for mask processing. When Masking is selected with the radio button Masking is performed.
T3 = sg.Tab("mask", [
[
sg.Radio(
'Rectangle',
"RADIO2",
key='-RECTANGLE_MASK-',
default=True,
size=(8, 1)
),
sg.Radio(
'Masking',
"RADIO2",
key='-MASKING-',
size=(8, 1)
)
],
[
sg.Checkbox(
"Blue",
size=(10, 1),
default=True,
key='-BLUE_MASK-',
enable_events=True
),
sg.Checkbox(
"Green",
size=(10, 1),
default=True,
key='-GREEN_MASK-',
enable_events=True
),
sg.Checkbox(
"Red",
size=(10, 1),
default=True,
key='-RED_MASK-',
enable_events=True
)
],
[
sg.Text(
'hsv',
size=(10, 1),
key='-HSV_MASK-',
enable_events=True
),
sg.Button('Blue', size=(10, 1)),
sg.Button('Green', size=(10, 1)),
sg.Button('Red', size=(10, 1))
],
[
sg.Checkbox(
'Hue Reverse',
size=(10, 1),
key='-Hue Reverse_MASK-',
enable_events=True
)
],
[
sg.Text('Hue', size=(10, 1), key='-Hue_MASK-'),
sg.Slider(
(0, 255),
0,
1,
orientation='h',
size=(19.4, 15),
key='-H_MIN SLIDER_MASK-',
enable_events=True
),
sg.Slider(
(1, 255),
125,
1,
orientation='h',
size=(19.4, 15),
key='-H_MAX SLIDER_MASK-',
enable_events=True
)
],
[
sg.Text('Saturation', size=(10, 1), key='-Saturation_MASK-'),
sg.Slider(
(0, 255),
50,
1,
orientation='h',
size=(19.4, 15),
key='-S_MIN SLIDER_MASK-',
enable_events=True
),
sg.Slider(
(1, 255),
255,
1,
orientation='h',
size=(19.4, 15),
key='-S_MAX SLIDER_MASK-',
enable_events=True
)
],
[
sg.Text('Value', size=(10, 1), key='-Value_MASK-'),
sg.Slider(
(0, 255),
50,
1,
orientation='h',
size=(19.4, 15),
key='-V_MIN SLIDER_MASK-',
enable_events=True
),
sg.Slider(
(1, 255),
255,
1,
orientation='h',
size=(19.4, 15),
key='-V_MAX SLIDER_MASK-',
enable_events=True
)
]
])
T4 = sg.Tab("Save", [
[
sg.Button('Write', size=(10, 1)),
sg.Radio(
'DIVX',
"RADIO1",
key='-DIVX-',
default=True,
size=(8, 1)
),
sg.Radio('MJPG', "RADIO1", key='-MJPG-', size=(8, 1)),
sg.Radio('GIF', "RADIO1", key='-GIF-', size=(8, 1))
],
[
sg.Text('Caption', size=(10, 1)),
sg.InputText(
size=(32, 50),
key='-CAPTION-',
enable_events=True
)
]
])
layout = [
[
sg.Text("Start", size=(8, 1)),
sg.Slider(
(0, self.total_count - 1),
0,
1,
orientation='h',
size=(45, 15),
key='-START FRAME SLIDER-',
enable_events=True
)
],
[
sg.Text("End ", size=(8, 1)),
sg.Slider(
(0, self.total_count - 1), self.total_count - 1,
1,
orientation='h',
size=(45, 15),
key='-END FRAME SLIDER-',
enable_events=True
)
],
[sg.Slider(
(0, self.total_count - 1),
0,
1,
orientation='h',
size=(50, 15),
key='-PROGRESS SLIDER-',
enable_events=True
)],
[
sg.Button('<<<', size=(5, 1)),
sg.Button('<<', size=(5, 1)),
sg.Button('<', size=(5, 1)),
sg.Button('Play / Stop', size=(9, 1)),
sg.Button('Reset', size=(7, 1)),
sg.Button('>', size=(5, 1)),
sg.Button('>>', size=(5, 1)),
sg.Button('>>>', size=(5, 1))
],
[
sg.Text("Speed", size=(6, 1)),
sg.Slider(
(0, 240),
10,
10,
orientation='h',
size=(19.4, 15),
key='-SPEED SLIDER-',
enable_events=True
),
sg.Text("Skip", size=(6, 1)),
sg.Slider(
(0, 300),
0,
1,
orientation='h',
size=(19.4, 15),
key='-SKIP SLIDER-',
enable_events=True
)
],
[sg.HorizontalSeparator()],
[
sg.TabGroup(
[[T1, T2, T3, T4]],
tab_background_color="#ccc",
selected_title_color="#fff",
selected_background_color="#444",
tab_location="topleft"
)
],
[sg.Output(size=(65, 5), key='-OUTPUT-')],
[sg.Button('Clear')]
]
#Generate Window
window = sg.Window('OpenCV Integration', layout, location=(0, 0))
#Display of video information
self.event, values = window.read(timeout=0)
print("The file has been read.")
print("File Path: " + str(self.fp))
print("fps: " + str(int(self.fps)))
print("width: " + str(self.width))
print("height: " + str(self.height))
print("frame count: " + str(int(self.total_count)))
#Main loop#########################################################
try:
while True:
#Loading GUI events
self.event, values = window.read(
timeout=values["-SPEED SLIDER-"]
)
#Show event in window
if self.event != "__TIMEOUT__":
print(self.event)
#Exit when the Exit button is pressed or when the window close button is pressed
if self.event in ('Exit', sg.WIN_CLOSED, None):
break
#Reload video
#Works when the start frame is set
if self.event == 'Reset':
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.s_frame)
self.frame_count = self.s_frame
window['-PROGRESS SLIDER-'].update(self.frame_count)
self.video_stabilization_flg = False
self.stab_prepare_flg = False
#Continue to reflect changes to Progress slider
continue
#Export video
if self.event == 'Write':
self.rec_flg = True
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.s_frame)
self.frame_count = self.s_frame
window['-PROGRESS SLIDER-'].update(self.frame_count)
if values["-GIF-"]:
images = []
else:
#Save as video
#Codec selection
#DIVX has a high compression rate
#MJEG can be analyzed with ImageJ
if values["-DIVX-"]:
codec = "DIVX"
elif values["-MJPG-"]:
codec = "MJPG"
fourcc = cv2.VideoWriter_fourcc(*codec)
out = cv2.VideoWriter(
str((
self.fp.parent / (self.fp.stem + '_' + codec + '.avi')
)),
fourcc,
self.fps,
(int(self.x2 - self.x1), int(self.y2 - self.y1))
)
continue
if self.event == 'Stabilization':
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.s_frame)
self.frame_count = self.s_frame
window['-PROGRESS SLIDER-'].update(self.frame_count)
self.play_count = int(self.e_frame - self.s_frame)
self.video_stabilization_flg = True
continue
#Frame operation################################################
#Priority is given if the slider is changed directly
if self.event == '-PROGRESS SLIDER-':
#Set the frame count to the progress bar
self.frame_count = int(values['-PROGRESS SLIDER-'])
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.frame_count)
if values['-PROGRESS SLIDER-'] > values['-END FRAME SLIDER-']:
window['-END FRAME SLIDER-'].update(
values['-PROGRESS SLIDER-'])
#If you change the start frame
if self.event == '-START FRAME SLIDER-':
self.s_frame = int(values['-START FRAME SLIDER-'])
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.s_frame)
self.frame_count = self.s_frame
window['-PROGRESS SLIDER-'].update(self.frame_count)
if values['-START FRAME SLIDER-'] > values['-END FRAME SLIDER-']:
window['-END FRAME SLIDER-'].update(
values['-START FRAME SLIDER-'])
self.e_frame = self.s_frame
#If you change the end frame
if self.event == '-END FRAME SLIDER-':
if values['-END FRAME SLIDER-'] < values['-START FRAME SLIDER-']:
window['-START FRAME SLIDER-'].update(
values['-END FRAME SLIDER-'])
self.s_frame = self.e_frame
#End frame settings
self.e_frame = int(values['-END FRAME SLIDER-'])
if self.event == '<<<':
self.frame_count = np.maximum(0, self.frame_count - 150)
window['-PROGRESS SLIDER-'].update(self.frame_count)
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.frame_count)
if self.event == '<<':
self.frame_count = np.maximum(0, self.frame_count - 30)
window['-PROGRESS SLIDER-'].update(self.frame_count)
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.frame_count)
if self.event == '<':
self.frame_count = np.maximum(0, self.frame_count - 1)
window['-PROGRESS SLIDER-'].update(self.frame_count)
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.frame_count)
if self.event == '>':
self.frame_count = self.frame_count + 1
window['-PROGRESS SLIDER-'].update(self.frame_count)
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.frame_count)
if self.event == '>>':
self.frame_count = self.frame_count + 30
window['-PROGRESS SLIDER-'].update(self.frame_count)
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.frame_count)
if self.event == '>>>':
self.frame_count = self.frame_count + 150
window['-PROGRESS SLIDER-'].update(self.frame_count)
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.frame_count)
#If the counter exceeds the end frame, restart from the start frame
if self.frame_count >= self.e_frame:
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.s_frame)
self.frame_count = self.s_frame
window['-PROGRESS SLIDER-'].update(self.frame_count)
continue
#Pause video loading with the stop button
if self.event == 'Play / Stop':
self.stop_flg = not self.stop_flg
#Unless the stop flag is set and an event occurs, count in
#Stop the operation
#If the stop button is pressed, the video processing will be stopped, but something
#If an event occurs, only update the image
#The same applies when operating the mouse
if(
(
self.stop_flg
and self.event == "__TIMEOUT__"
and self.mouse_flg is False
)
):
window['-PROGRESS SLIDER-'].update(self.frame_count)
continue
#Skip frames
if not self.stop_flg and values['-SKIP SLIDER-'] != 0:
self.frame_count += values["-SKIP SLIDER-"]
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.frame_count)
#Load frame##############################################
self.ret, self.frame = self.cap.read()
self.valid_frame = int(self.frame_count - self.s_frame)
#Self when the last frame is over.s_Resume from frame
if not self.ret:
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.s_frame)
self.frame_count = self.s_frame
continue
#Describe the processing for the frame after that##################################
#First carry out processing for the entire frame##############################
#resize
self.width = int(self.org_width * values['-RESIZE SLIDER-'])
self.height = int(self.org_height * values['-RESIZE SLIDER-'])
self.frame = cv2.resize(self.frame, (self.width, self.height))
if self.event == '-RESIZE SLIDER-':
self.x1 = self.y1 = 0
self.x2 = self.width
self.y2 = self.height
#Perform processing for ROI##########################################
if self.roi_flg:
self.frame_roi = self.frame[
self.y1:self.y2, self.x1:self.x2, :
]
RGB mask processing
Describe the mask processing. The color is specified in RGB instead of HSV. After specifying the color in RGB, you can specify the mask range in V (brightness) of HSV. In this case, Hue should specify the entire range 0-255 and change only V.
#Processing to MASK images##################################################################
if values['-MASKING-']:
#RGB separation
self.mask = np.copy(self.frame_roi)
if not values['-BLUE_MASK-']:
self.mask[:, :, 0] = 0
if not values['-GREEN_MASK-']:
self.mask[:, :, 1] = 0
if not values['-RED_MASK-']:
self.mask[:, :, 2] = 0
HSV mask processing
Describes mask processing in HSV. The Red, Green, and Blue buttons allow you to select each color within a certain range. You can mask the object by adjusting the threshold while watching the original video and the mask video.
if self.event == 'Blue':
window['-H_MIN SLIDER_MASK-'].update(70)
window['-H_MAX SLIDER_MASK-'].update(110)
window['-S_MIN SLIDER_MASK-'].update(70)
window['-S_MAX SLIDER_MASK-'].update(255)
window['-V_MIN SLIDER_MASK-'].update(0)
window['-V_MAX SLIDER_MASK-'].update(255)
window['-Hue Reverse_MASK-'].update(False)
if self.event == 'Green':
window['-H_MIN SLIDER_MASK-'].update(20)
window['-H_MAX SLIDER_MASK-'].update(70)
window['-S_MIN SLIDER_MASK-'].update(70)
window['-S_MAX SLIDER_MASK-'].update(255)
window['-V_MIN SLIDER_MASK-'].update(0)
window['-V_MAX SLIDER_MASK-'].update(255)
window['-Hue Reverse_MASK-'].update(False)
if self.event == 'Red':
window['-H_MIN SLIDER_MASK-'].update(20)
window['-H_MAX SLIDER_MASK-'].update(110)
window['-S_MIN SLIDER_MASK-'].update(70)
window['-S_MAX SLIDER_MASK-'].update(255)
window['-V_MIN SLIDER_MASK-'].update(0)
window['-V_MAX SLIDER_MASK-'].update(255)
window['-Hue Reverse_MASK-'].update(True)
self.mask = hsv(
self.mask,
values['-H_MAX SLIDER_MASK-'],
values['-H_MIN SLIDER_MASK-'],
values['-S_MAX SLIDER_MASK-'],
values['-S_MIN SLIDER_MASK-'],
values['-V_MAX SLIDER_MASK-'],
values['-V_MIN SLIDER_MASK-'],
values['-Hue Reverse_MASK-']
)
#Grayscale
self.mask = cv2.cvtColor(
self.mask,
cv2.COLOR_BGR2GRAY
)
#Blur
if values['-BLUR-']:
self.frame_roi = cv2.GaussianBlur(
self.frame_roi, (21, 21), values['-BLUR SLIDER-']
)
if values['-GRAY-']:
self.frame_roi = cv2.cvtColor(
self.frame_roi,
cv2.COLOR_BGR2GRAY
)
self.frame_roi = cv2.cvtColor(
self.frame_roi,
cv2.COLOR_GRAY2BGR
)
Apply processing only to the mask range within the ROI. I'm using cv2.bitwise_not nested, but there may be a better way to do this.
if values['-MASKING-']:
# frame_Apply mask inside roi
#Only the mask processing part is framed_Change to roi
self.frame_roi = cv2.bitwise_not(
cv2.bitwise_not(self.frame_roi),
self.frame[self.y1:self.y2, self.x1:self.x2, :],
mask=self.mask
)
#Return processed ROI to frame
self.frame[self.y1:self.y2, self.x1:self.x2, :] = self.frame_roi
#Save video
if self.rec_flg:
#Cut out roi again after image stabilization
self.frame_roi = self.frame[
self.y1:self.y2, self.x1:self.x2, :
]
if values["-GIF-"]:
images.append(
Image.fromarray(
cv2.cvtColor(
self.frame_roi, cv2.COLOR_BGR2RGB
)
)
)
else:
out.write(self.frame_roi)
#Display during saving
cv2.putText(
self.frame,
str("Now Recording"),
(20, 60),
cv2.FONT_HERSHEY_SIMPLEX,
0.5,
(10, 10, 255),
1,
cv2.LINE_AA
)
# e_Finish when it becomes a frame
if self.frame_count >= self.e_frame - values["-SKIP SLIDER-"] - 1:
if values["-GIF-"]:
images[0].save(
str((self.fp.parent / (self.fp.stem + '.gif'))),
save_all=True,
append_images=images[1:],
optimize=False,
duration=1000 // self.fps,
loop=0
)
else:
out.release()
self.rec_flg = False
#Display of number of frames and elapsed seconds
cv2.putText(
self.frame, str("framecount: {0:.0f}".format(self.frame_count)), (
15, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (240, 230, 0), 1, cv2.LINE_AA
)
cv2.putText(
self.frame, str("time: {0:.1f} sec".format(
self.frame_count / self.fps)), (15, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (240, 230, 0), 1, cv2.LINE_AA
)
#When performing calculation to ROI or while pressing the left mouse button
#Draw a rectangle
if self.roi_flg or self.mouse_flg:
cv2.rectangle(
self.frame,
(self.x1, self.y1),
(self.x2 - 1, self.y2 - 1),
(128, 128, 128)
)
The mask image is displayed only when Masking is selected.
#Display image
cv2.imshow("Movie", self.frame)
if values['-MASKING-']:
cv2.imshow("Mask", cv2.cvtColor(self.mask, cv2.COLOR_GRAY2BGR))
cv2.setWindowProperty("Mask", cv2.WND_PROP_VISIBLE, 0)
elif not values['-MASKING-'] and cv2.getWindowProperty("Mask", cv2.WND_PROP_VISIBLE):
cv2.destroyWindow("Mask")
if self.stop_flg:
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.frame_count)
else:
self.frame_count += 1
window['-PROGRESS SLIDER-'].update(self.frame_count + 1)
#Other processing###############################################
#Clear log window
if self.event == 'Clear':
window['-OUTPUT-'].update('')
finally:
cv2.destroyWindow("Movie")
cv2.destroyWindow("Mask")
self.cap.release()
window.close()
if __name__ == '__main__':
Main().run()
Reference link
Let's recognize red objects with python Wikipedia: HSV color space Alpha blending and masking images with Python, OpenCV, NumPy
Recommended Posts