如何实时检测对象并自动跟踪它，而不是用户必须在要跟踪的对象周围绘制边界框？
How to detect an object real time and track it automatically, instead of user having to draw a bounding box around the object to be tracked?
我有以下代码，用户可以按p暂停视频，在要跟踪的对象周围绘制一个边界框，然后按Enter（回车return）进行跟踪视频源中的那个对象：
import cv2
import sys
major_ver, minor_ver, subminor_ver = cv2.__version__.split('.')
if __name__ == '__main__' :
# Set up tracker.
tracker_types = ['BOOSTING', 'MIL','KCF', 'TLD', 'MEDIANFLOW', 'GOTURN', 'MOSSE', 'CSRT']
tracker_type = tracker_types[1]
if int(minor_ver) < 3:
tracker = cv2.Tracker_create(tracker_type)
else:
if tracker_type == 'BOOSTING':
tracker = cv2.TrackerBoosting_create()
if tracker_type == 'MIL':
tracker = cv2.TrackerMIL_create()
if tracker_type == 'KCF':
tracker = cv2.TrackerKCF_create()
if tracker_type == 'TLD':
tracker = cv2.TrackerTLD_create()
if tracker_type == 'MEDIANFLOW':
tracker = cv2.TrackerMedianFlow_create()
if tracker_type == 'GOTURN':
tracker = cv2.TrackerGOTURN_create()
if tracker_type == 'MOSSE':
tracker = cv2.TrackerMOSSE_create()
if tracker_type == "CSRT":
tracker = cv2.TrackerCSRT_create()
# Read video
video = cv2.VideoCapture(0) # 0 means webcam. Otherwise if you want to use a video file, replace 0 with "video_file.MOV")
# Exit if video not opened.
if not video.isOpened():
print ("Could not open video")
sys.exit()
while True:
# Read first frame.
ok, frame = video.read()
if not ok:
print ('Cannot read video file')
sys.exit()
# Retrieve an image and Display it.
if((0xFF & cv2.waitKey(10))==ord('p')): # Press key `p` to pause the video to start tracking
break
cv2.namedWindow("Image", cv2.WINDOW_NORMAL)
cv2.imshow("Image", frame)
cv2.destroyWindow("Image");
# select the bounding box
bbox = (287, 23, 86, 320)
# Uncomment the line below to select a different bounding box
bbox = cv2.selectROI(frame, False)
# Initialize tracker with first frame and bounding box
ok = tracker.init(frame, bbox)
while True:
# Read a new frame
ok, frame = video.read()
if not ok:
break
# Start timer
timer = cv2.getTickCount()
# Update tracker
ok, bbox = tracker.update(frame)
# Calculate Frames per second (FPS)
fps = cv2.getTickFrequency() / (cv2.getTickCount() - timer);
# Draw bounding box
if ok:
# Tracking success
p1 = (int(bbox[0]), int(bbox[1]))
p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3]))
cv2.rectangle(frame, p1, p2, (255,0,0), 2, 1)
else :
# Tracking failure
cv2.putText(frame, "Tracking failure detected", (100,80), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,0,255),2)
# Display tracker type on frame
cv2.putText(frame, tracker_type + " Tracker", (100,20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50,170,50),2);
# Display FPS on frame
cv2.putText(frame, "FPS : " + str(int(fps)), (100,50), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50,170,50), 2);
# Display result
cv2.imshow("Tracking", frame)
# Exit if ESC pressed
k = cv2.waitKey(1) & 0xff
if k == 27 : break
现在，不是让用户暂停视频并在对象周围绘制边界框，而是如何让它可以自动检测我感兴趣的特定对象（在我的例子中是牙刷）每当它在视频提要中介绍时，然后跟踪它？
我找到了 this 篇文章，其中讨论了我们如何使用 ImageAI 和 Yolo 检测视频中的对象。
from imageai.Detection import VideoObjectDetection
import os
import cv2
execution_path = os.getcwd()
camera = cv2.VideoCapture(0)
detector = VideoObjectDetection()
detector.setModelTypeAsYOLOv3()
detector.setModelPath(os.path.join(execution_path , "yolo.h5"))
detector.loadModel()
video_path = detector.detectObjectsFromVideo(camera_input=camera,
output_file_path=os.path.join(execution_path, "camera_detected_1")
, frames_per_second=29, log_progress=True)
print(video_path)
现在，Yolo 确实检测了牙刷，它是它默认可以检测的 80 个奇数对象之一。但是，这篇文章有两点对我来说不是理想的解决方案：
此方法首先分析每个视频帧（每帧大约需要 1-2 秒，因此大约需要 1 分钟来分析来自网络摄像头的 2-3 秒视频流），并保存检测到的视频在一个单独的视频文件中。然而，我想实时检测网络摄像头视频中的牙刷。有解决办法吗？
正在使用的 Yolo v3 模型可以检测所有 80 个物体，但我只想检测 2 或 3 个物体——牙刷、拿着牙刷的人和背景，如果需要的话。那么，有没有一种方法可以通过只选择这 2 或 3 个对象来检测来降低模型权重？
我想借助 this article 来回答这个问题，我以前也用过它，遇到了与你类似的问题。以下是建议：
- 运行 YOLOv3 与 darknet framework 这将提高性能。
- 在您的代码片段中，它似乎不允许您决定网络输入的宽度和高度，所以我不知道您对它们使用了什么。减小网络宽度和高度会提高速度，但相反会降低准确性。
- YOLOv3 针对 80 个对象进行了训练，但您只需要其中的一部分。我之前也只需要我项目中的汽车。不幸的是，你不能操纵已经训练好的权重文件，也不能很好地训练你的对象。
- 我之前也尝试过的另一种方法是我将YOLOv3转移到另一个线程并且我也没有将yolo应用于所有帧。我只应用了其中的一些，例如：每 10 帧中的 1 帧。这对我也很有帮助。
- 或者您可以选择更好的 cpu pc :)
如果你想要一个快速简单的解决方案，你可以使用一个更轻量级的 yolo 文件。你可以从这个网站获取权重和配置文件（它们成对出现并且必须一起使用）：https://pjreddie.com/darknet/yolo/（别担心，它看起来很草图但没问题）
使用较小的网络可以获得更高的 fps，但准确度也会降低。如果这是您愿意接受的权衡，那么这是最简单的事情。
这是一些检测牙刷的代码。第一个文件只是一个 class 文件，以帮助更无缝地使用 Yolo 网络。第二个是打开 VideoCapture 并将图像提供给网络的“主”文件。
yolo.py
import cv2
import numpy as np
class Yolo:
def __init__(self, cfg, weights, names, conf_thresh, nms_thresh, use_cuda = False):
# save thresholds
self.ct = conf_thresh;
self.nmst = nms_thresh;
# create net
self.net = cv2.dnn.readNet(weights, cfg);
print("Finished: " + str(weights));
self.classes = [];
file = open(names, 'r');
for line in file:
self.classes.append(line.strip());
# use gpu + CUDA to speed up detections
if use_cuda:
self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA);
self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA);
# get output names
layer_names = self.net.getLayerNames();
self.output_layers = [layer_names[i[0]-1] for i in self.net.getUnconnectedOutLayers()];
# runs detection on the image and draws on it
def detect(self, img, target_id):
# get detection stuff
b, c, ids, idxs = self.get_detection_data(img, target_id);
# draw result
img = self.draw(img, b, c, ids, idxs);
return img, len(idxs);
# returns boxes, confidences, class_ids, and indexes (indices?)
def get_detection_data(self, img, target_id):
# get output
layer_outputs = self.get_inf(img);
# get dims
height, width = img.shape[:2];
# filter thresholds and target
b, c, ids, idxs = self.thresh(layer_outputs, width, height, target_id);
return b, c, ids, idxs;
# runs the network on an image
def get_inf(self, img):
# construct a blob
blob = cv2.dnn.blobFromImage(img, 1 / 255.0, (416,416), swapRB=True, crop=False);
# get response
self.net.setInput(blob);
layer_outputs = self.net.forward(self.output_layers);
return layer_outputs;
# filters the layer output by conf, nms and id
def thresh(self, layer_outputs, width, height, target_id):
# some lists
boxes = [];
confidences = [];
class_ids = [];
# each layer outputs
for output in layer_outputs:
for detection in output:
# get id and confidence
scores = detection[5:];
class_id = np.argmax(scores);
confidence = scores[class_id];
# filter out low confidence
if confidence > self.ct and class_id == target_id:
# scale bounding box back to the image size
box = detection[0:4] * np.array([width, height, width, height]);
(cx, cy, w, h) = box.astype('int');
# grab the top-left corner of the box
tx = int(cx - (w / 2));
ty = int(cy - (h / 2));
# update lists
boxes.append([tx,ty,int(w),int(h)]);
confidences.append(float(confidence));
class_ids.append(class_id);
# apply NMS
idxs = cv2.dnn.NMSBoxes(boxes, confidences, self.ct, self.nmst);
return boxes, confidences, class_ids, idxs;
# draw detections on image
def draw(self, img, boxes, confidences, class_ids, idxs):
# check for zero
if len(idxs) > 0:
# loop over indices
for i in idxs.flatten():
# extract the bounding box coords
(x,y) = (boxes[i][0], boxes[i][1]);
(w,h) = (boxes[i][2], boxes[i][3]);
# draw a box
cv2.rectangle(img, (x,y), (x+w,y+h), (0,0,255), 2);
# draw text
text = "{}: {:.4}".format(self.classes[class_ids[i]], confidences[i]);
cv2.putText(img, text, (x, y-5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255), 2);
return img;
main.py
import cv2
import numpy as np
# this is the "yolo.py" file, I assume it's in the same folder as this program
from yolo import Yolo
# these are the filepaths of the yolo files
weights = "yolov3-tiny.weights";
config = "yolov3-tiny.cfg";
labels = "yolov3.txt";
# init yolo network
target_class_id = 79; # toothbrush
conf_thresh = 0.4; # less == more boxes (but more false positives)
nms_thresh = 0.4; # less == more boxes (but more overlap)
net = Yolo(config, weights, labels, conf_thresh, nms_thresh);
# open video capture
cap = cv2.VideoCapture(0);
# loop
done = False;
while not done:
# get frame
ret, frame = cap.read();
if not ret:
done = cv2.waitKey(1) == ord('q');
continue;
# do detection
frame, _ = net.detect(frame, target_class_id);
# show
cv2.imshow("Marked", frame);
done = cv2.waitKey(1) == ord('q');
如果您不想使用较轻的权重文件，有几个选项供您选择。
如果您有 Nvidia GPU，则可以使用 CUDA 大幅 提高帧率。即使是普通的 nvidia gpu 也比仅在 cpu.
上 运行ning 快几倍
绕过不断 运行ning 检测成本的常见策略是仅使用它来初始获取目标。您可以使用神经网络的检测来初始化您的对象跟踪器，类似于人在对象周围绘制边界框。对象跟踪器速度更快，无需每帧不断进行全面检测。
如果您 运行 在单独的线程中进行 Yolo 和对象跟踪，那么您可以 运行 尽可能快地使用相机。您需要存储帧的历史记录，以便当 Yolo 线程完成一个帧时，您可以检查旧帧以查看您是否已经在跟踪对象，这样您就可以在相应的帧上快速启动对象跟踪器-转发它让它赶上。这个程序并不简单，您需要确保正确管理线程之间的数据。这是一个很好的练习，可以让你熟悉多线程，这是编程的一大步。
我有以下代码，用户可以按p暂停视频，在要跟踪的对象周围绘制一个边界框，然后按Enter（回车return）进行跟踪视频源中的那个对象：
import cv2
import sys
major_ver, minor_ver, subminor_ver = cv2.__version__.split('.')
if __name__ == '__main__' :
# Set up tracker.
tracker_types = ['BOOSTING', 'MIL','KCF', 'TLD', 'MEDIANFLOW', 'GOTURN', 'MOSSE', 'CSRT']
tracker_type = tracker_types[1]
if int(minor_ver) < 3:
tracker = cv2.Tracker_create(tracker_type)
else:
if tracker_type == 'BOOSTING':
tracker = cv2.TrackerBoosting_create()
if tracker_type == 'MIL':
tracker = cv2.TrackerMIL_create()
if tracker_type == 'KCF':
tracker = cv2.TrackerKCF_create()
if tracker_type == 'TLD':
tracker = cv2.TrackerTLD_create()
if tracker_type == 'MEDIANFLOW':
tracker = cv2.TrackerMedianFlow_create()
if tracker_type == 'GOTURN':
tracker = cv2.TrackerGOTURN_create()
if tracker_type == 'MOSSE':
tracker = cv2.TrackerMOSSE_create()
if tracker_type == "CSRT":
tracker = cv2.TrackerCSRT_create()
# Read video
video = cv2.VideoCapture(0) # 0 means webcam. Otherwise if you want to use a video file, replace 0 with "video_file.MOV")
# Exit if video not opened.
if not video.isOpened():
print ("Could not open video")
sys.exit()
while True:
# Read first frame.
ok, frame = video.read()
if not ok:
print ('Cannot read video file')
sys.exit()
# Retrieve an image and Display it.
if((0xFF & cv2.waitKey(10))==ord('p')): # Press key `p` to pause the video to start tracking
break
cv2.namedWindow("Image", cv2.WINDOW_NORMAL)
cv2.imshow("Image", frame)
cv2.destroyWindow("Image");
# select the bounding box
bbox = (287, 23, 86, 320)
# Uncomment the line below to select a different bounding box
bbox = cv2.selectROI(frame, False)
# Initialize tracker with first frame and bounding box
ok = tracker.init(frame, bbox)
while True:
# Read a new frame
ok, frame = video.read()
if not ok:
break
# Start timer
timer = cv2.getTickCount()
# Update tracker
ok, bbox = tracker.update(frame)
# Calculate Frames per second (FPS)
fps = cv2.getTickFrequency() / (cv2.getTickCount() - timer);
# Draw bounding box
if ok:
# Tracking success
p1 = (int(bbox[0]), int(bbox[1]))
p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3]))
cv2.rectangle(frame, p1, p2, (255,0,0), 2, 1)
else :
# Tracking failure
cv2.putText(frame, "Tracking failure detected", (100,80), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,0,255),2)
# Display tracker type on frame
cv2.putText(frame, tracker_type + " Tracker", (100,20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50,170,50),2);
# Display FPS on frame
cv2.putText(frame, "FPS : " + str(int(fps)), (100,50), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50,170,50), 2);
# Display result
cv2.imshow("Tracking", frame)
# Exit if ESC pressed
k = cv2.waitKey(1) & 0xff
if k == 27 : break
现在，不是让用户暂停视频并在对象周围绘制边界框，而是如何让它可以自动检测我感兴趣的特定对象（在我的例子中是牙刷）每当它在视频提要中介绍时，然后跟踪它？
我找到了 this 篇文章，其中讨论了我们如何使用 ImageAI 和 Yolo 检测视频中的对象。
from imageai.Detection import VideoObjectDetection
import os
import cv2
execution_path = os.getcwd()
camera = cv2.VideoCapture(0)
detector = VideoObjectDetection()
detector.setModelTypeAsYOLOv3()
detector.setModelPath(os.path.join(execution_path , "yolo.h5"))
detector.loadModel()
video_path = detector.detectObjectsFromVideo(camera_input=camera,
output_file_path=os.path.join(execution_path, "camera_detected_1")
, frames_per_second=29, log_progress=True)
print(video_path)
现在，Yolo 确实检测了牙刷，它是它默认可以检测的 80 个奇数对象之一。但是，这篇文章有两点对我来说不是理想的解决方案：
此方法首先分析每个视频帧（每帧大约需要 1-2 秒，因此大约需要 1 分钟来分析来自网络摄像头的 2-3 秒视频流），并保存检测到的视频在一个单独的视频文件中。然而，我想实时检测网络摄像头视频中的牙刷。有解决办法吗？
正在使用的 Yolo v3 模型可以检测所有 80 个物体，但我只想检测 2 或 3 个物体——牙刷、拿着牙刷的人和背景，如果需要的话。那么，有没有一种方法可以通过只选择这 2 或 3 个对象来检测来降低模型权重？
我想借助 this article 来回答这个问题，我以前也用过它，遇到了与你类似的问题。以下是建议：
- 运行 YOLOv3 与 darknet framework 这将提高性能。
- 在您的代码片段中，它似乎不允许您决定网络输入的宽度和高度，所以我不知道您对它们使用了什么。减小网络宽度和高度会提高速度，但相反会降低准确性。
- YOLOv3 针对 80 个对象进行了训练，但您只需要其中的一部分。我之前也只需要我项目中的汽车。不幸的是，你不能操纵已经训练好的权重文件，也不能很好地训练你的对象。
- 我之前也尝试过的另一种方法是我将YOLOv3转移到另一个线程并且我也没有将yolo应用于所有帧。我只应用了其中的一些，例如：每 10 帧中的 1 帧。这对我也很有帮助。
- 或者您可以选择更好的 cpu pc :)
如果你想要一个快速简单的解决方案，你可以使用一个更轻量级的 yolo 文件。你可以从这个网站获取权重和配置文件（它们成对出现并且必须一起使用）：https://pjreddie.com/darknet/yolo/（别担心，它看起来很草图但没问题）
使用较小的网络可以获得更高的 fps，但准确度也会降低。如果这是您愿意接受的权衡，那么这是最简单的事情。
这是一些检测牙刷的代码。第一个文件只是一个 class 文件，以帮助更无缝地使用 Yolo 网络。第二个是打开 VideoCapture 并将图像提供给网络的“主”文件。
yolo.py
import cv2
import numpy as np
class Yolo:
def __init__(self, cfg, weights, names, conf_thresh, nms_thresh, use_cuda = False):
# save thresholds
self.ct = conf_thresh;
self.nmst = nms_thresh;
# create net
self.net = cv2.dnn.readNet(weights, cfg);
print("Finished: " + str(weights));
self.classes = [];
file = open(names, 'r');
for line in file:
self.classes.append(line.strip());
# use gpu + CUDA to speed up detections
if use_cuda:
self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA);
self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA);
# get output names
layer_names = self.net.getLayerNames();
self.output_layers = [layer_names[i[0]-1] for i in self.net.getUnconnectedOutLayers()];
# runs detection on the image and draws on it
def detect(self, img, target_id):
# get detection stuff
b, c, ids, idxs = self.get_detection_data(img, target_id);
# draw result
img = self.draw(img, b, c, ids, idxs);
return img, len(idxs);
# returns boxes, confidences, class_ids, and indexes (indices?)
def get_detection_data(self, img, target_id):
# get output
layer_outputs = self.get_inf(img);
# get dims
height, width = img.shape[:2];
# filter thresholds and target
b, c, ids, idxs = self.thresh(layer_outputs, width, height, target_id);
return b, c, ids, idxs;
# runs the network on an image
def get_inf(self, img):
# construct a blob
blob = cv2.dnn.blobFromImage(img, 1 / 255.0, (416,416), swapRB=True, crop=False);
# get response
self.net.setInput(blob);
layer_outputs = self.net.forward(self.output_layers);
return layer_outputs;
# filters the layer output by conf, nms and id
def thresh(self, layer_outputs, width, height, target_id):
# some lists
boxes = [];
confidences = [];
class_ids = [];
# each layer outputs
for output in layer_outputs:
for detection in output:
# get id and confidence
scores = detection[5:];
class_id = np.argmax(scores);
confidence = scores[class_id];
# filter out low confidence
if confidence > self.ct and class_id == target_id:
# scale bounding box back to the image size
box = detection[0:4] * np.array([width, height, width, height]);
(cx, cy, w, h) = box.astype('int');
# grab the top-left corner of the box
tx = int(cx - (w / 2));
ty = int(cy - (h / 2));
# update lists
boxes.append([tx,ty,int(w),int(h)]);
confidences.append(float(confidence));
class_ids.append(class_id);
# apply NMS
idxs = cv2.dnn.NMSBoxes(boxes, confidences, self.ct, self.nmst);
return boxes, confidences, class_ids, idxs;
# draw detections on image
def draw(self, img, boxes, confidences, class_ids, idxs):
# check for zero
if len(idxs) > 0:
# loop over indices
for i in idxs.flatten():
# extract the bounding box coords
(x,y) = (boxes[i][0], boxes[i][1]);
(w,h) = (boxes[i][2], boxes[i][3]);
# draw a box
cv2.rectangle(img, (x,y), (x+w,y+h), (0,0,255), 2);
# draw text
text = "{}: {:.4}".format(self.classes[class_ids[i]], confidences[i]);
cv2.putText(img, text, (x, y-5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255), 2);
return img;
main.py
import cv2
import numpy as np
# this is the "yolo.py" file, I assume it's in the same folder as this program
from yolo import Yolo
# these are the filepaths of the yolo files
weights = "yolov3-tiny.weights";
config = "yolov3-tiny.cfg";
labels = "yolov3.txt";
# init yolo network
target_class_id = 79; # toothbrush
conf_thresh = 0.4; # less == more boxes (but more false positives)
nms_thresh = 0.4; # less == more boxes (but more overlap)
net = Yolo(config, weights, labels, conf_thresh, nms_thresh);
# open video capture
cap = cv2.VideoCapture(0);
# loop
done = False;
while not done:
# get frame
ret, frame = cap.read();
if not ret:
done = cv2.waitKey(1) == ord('q');
continue;
# do detection
frame, _ = net.detect(frame, target_class_id);
# show
cv2.imshow("Marked", frame);
done = cv2.waitKey(1) == ord('q');
如果您不想使用较轻的权重文件，有几个选项供您选择。
如果您有 Nvidia GPU，则可以使用 CUDA 大幅 提高帧率。即使是普通的 nvidia gpu 也比仅在 cpu.
上 运行ning 快几倍绕过不断 运行ning 检测成本的常见策略是仅使用它来初始获取目标。您可以使用神经网络的检测来初始化您的对象跟踪器，类似于人在对象周围绘制边界框。对象跟踪器速度更快，无需每帧不断进行全面检测。
如果您 运行 在单独的线程中进行 Yolo 和对象跟踪，那么您可以 运行 尽可能快地使用相机。您需要存储帧的历史记录，以便当 Yolo 线程完成一个帧时，您可以检查旧帧以查看您是否已经在跟踪对象，这样您就可以在相应的帧上快速启动对象跟踪器-转发它让它赶上。这个程序并不简单，您需要确保正确管理线程之间的数据。这是一个很好的练习，可以让你熟悉多线程，这是编程的一大步。