drivers/vision/npu/npu-queue.c

/*
 * Samsung Exynos SoC series NPU driver
 *
 * Copyright (c) 2017 Samsung Electronics Co., Ltd.
 *              http://www.samsung.com/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include "npu-queue.h"
#include "npu-log.h"

/*
 * queue: npu_queue instance strucure
 * memory: Reference to memory management helper
 * lock: Session-wide mutex to protect access to vb_queue
 */

int npu_queue_open(struct npu_queue *queue,
	struct npu_memory *memory,
	struct mutex *lock)
{
	int ret = 0;
	struct vb_queue *inq, *otq;

	queue->qops = &npu_queue_ops;
	inq = &queue->inqueue;
	otq = &queue->otqueue;
	inq->private_data = queue;
	otq->private_data = queue;

	ret = vb_queue_init(inq, memory->dev, memory->vb2_mem_ops, &vb_ops, lock, VS4L_DIRECTION_IN);
	if (ret) {
		npu_err("fail(%d) in vb_queue_init\n", ret);
		goto p_err;
	}

	ret = vb_queue_init(otq, memory->dev, memory->vb2_mem_ops, &vb_ops, lock, VS4L_DIRECTION_OT);
	if (ret) {
		npu_err("fail(%d) in vb_queue_init\n", ret);
		goto p_err;
	}
p_err:
	return ret;
}

int npu_queue_s_format(struct npu_queue *queue, struct vs4l_format_list *f)
{
	int ret = 0;
	struct vb_queue *q, *inq, *otq;

	BUG_ON(!queue);
	BUG_ON(!f);

	inq = &queue->inqueue;
	otq = &queue->otqueue;

	if (f->direction == VS4L_DIRECTION_IN)
		q = inq;
	else
		q = otq;

	ret = vb_queue_s_format(q, f);
	if (ret) {
		npu_err("fail(%d) in vb_queue_s_format\n", ret);
		goto p_err;
	}

	npu_trace("dir(%u), q->state = %lu\n", f->direction, q->state);
	ret = CALL_QOPS(queue, format, f);
	if (ret) {
		npu_err("fail(%d) in CALL_QOPS(format)\n", ret);
		goto p_err;
	}

p_err:
	return ret;
}

int npu_queue_start(struct npu_queue *queue)
{
	int ret = 0;
	struct vb_queue *inq, *otq;

	inq = &queue->inqueue;
	otq = &queue->otqueue;

	ret = vb_queue_start(inq);
	if (ret) {
		npu_err("fail(%d) in vb_queue_init\n", ret);
		goto p_err;
	}

	ret = vb_queue_start(otq);
	if (ret) {
		npu_err("fail(%d) in vb_queue_init\n", ret);
		goto p_err;
	}

	ret = CALL_QOPS(queue, start);
	if (ret) {
		npu_err("fail(%d) in CALL_QOPS(start)\n", ret);
		goto p_err;
	}

p_err:
	return ret;
}

int npu_queue_streamoff(struct npu_queue *queue)
{
	int ret = 0;

	ret = CALL_QOPS(queue, streamoff);
	if (ret) {
		npu_err("fail(%d) in CALL_QOPS(stop)\n", ret);
		goto p_err;
	}
p_err:
	return ret;
}

int npu_queue_stop(struct npu_queue *queue, int is_forced)
{
	int ret = 0;
	struct vb_queue *inq, *otq;

	inq = &queue->inqueue;
	otq = &queue->otqueue;

	if (!test_bit(VB_QUEUE_STATE_START, &inq->state) && test_bit(VB_QUEUE_STATE_FORMAT, &inq->state)) {
		npu_info("already npu_queue_stop inq done\n");
		goto p_err;
	}

	if (!test_bit(VB_QUEUE_STATE_START, &otq->state) && test_bit(VB_QUEUE_STATE_FORMAT, &otq->state)) {
		npu_info("already npu_queue_stop otq done\n");
		goto p_err;
	}

	if (!is_forced)
		ret = vb_queue_stop(inq);
	else
		ret = vb_queue_stop_forced(inq);

	if (ret) {
		npu_err("fail(%d) in vb_queue_stop%s(inq)\n", ret, (is_forced)?"_forced":"");
		goto p_err;
	}

	if (!is_forced)
		ret = vb_queue_stop(otq);
	else
		ret = vb_queue_stop_forced(otq);

	if (ret) {
		npu_err("fail(%d) in vb_queue_stop%s(otq)\n", ret, (is_forced)?"_forced":"");
		goto p_err;
	}

	ret = CALL_QOPS(queue, stop);
	if (ret) {
		npu_err("fail(%d) in CALL_QOPS(stop)\n", ret);
		goto p_err;
	}

p_err:
	return ret;
}

int npu_queue_poll(struct npu_queue *queue, struct file *file, poll_table *poll)
{
	int ret = 0;
	struct vb_queue *inq, *otq;
	unsigned long events;
	u32 done_cnt;

	BUG_ON(!queue);
	BUG_ON(!file);
	BUG_ON(!poll);

	events = poll_requested_events(poll);

	inq = &queue->inqueue;
	otq = &queue->otqueue;

	if (events & POLLOUT) {
		done_cnt = (u32)atomic_read(&inq->done_count);
		if (list_empty(&inq->done_list)) {
			poll_wait(file, &inq->done_wq, poll);
		}

		if (!list_empty(&inq->done_list)) {
			ret |= POLLOUT | POLLWRNORM;
		}
	}

	if (events & POLLIN) {
		done_cnt = (u32)atomic_read(&otq->done_count);
		if (list_empty(&otq->done_list)) {
			poll_wait(file, &otq->done_wq, poll);
		}

		if (!list_empty(&otq->done_list)) {
			ret |= POLLIN | POLLWRNORM;
		}
	}

	return ret;
}

int npu_queue_qbuf(struct npu_queue *queue, struct vs4l_container_list *c)
{
	int ret = 0;
	struct vb_queue *q, *inq, *otq;
	struct vb_bundle *invb, *otvb;
	struct vb_bundle *tmp_vb;

	inq = &queue->inqueue;
	otq = &queue->otqueue;

	if (c->direction == VS4L_DIRECTION_IN)
		q = inq;
	else
		q = otq;


	ret = vb_queue_qbuf(q, c);
	if (ret) {
		npu_err("vb_queue_qbuf is fail(%d)\n", ret);
		goto p_err;
	}

	invb = NULL;
	list_for_each_entry(tmp_vb, &inq->queued_list, queued_entry) {
		if (tmp_vb->state == VB_BUF_STATE_QUEUED) {
			invb = tmp_vb;
			break;
		}
	}

	otvb = NULL;
	list_for_each_entry(tmp_vb, &otq->queued_list, queued_entry) {
		if (tmp_vb->state == VB_BUF_STATE_QUEUED) {
			otvb = tmp_vb;
			break;
		}
	}

	if (invb == NULL || otvb == NULL) {
		goto p_err;
	}

	vb_queue_process(inq, invb);
	vb_queue_process(otq, otvb);

	ret = CALL_QOPS(queue, queue, &invb->clist, &otvb->clist);
	if (ret) {
		npu_err("fail(%d) in CALL_QOPS(queue)\n", ret);
		goto p_err;
	}
p_err:
	return ret;
}

int npu_queue_prepare(struct npu_queue *queue, struct vs4l_container_list *c)
{
	int ret = 0;
	struct vb_queue *q, *inq, *otq;

	inq = &queue->inqueue;
	otq = &queue->otqueue;

	if (c->direction == VS4L_DIRECTION_IN)
		q = inq;
	else
		q = otq;

	ret = vb_queue_prepare(q, c);
	if (ret) {
		npu_err("vb_queue_prepare is fail(%d)\n", ret);
		goto p_err;
	}
p_err:
	return ret;
}

int npu_queue_unprepare(struct npu_queue *queue, struct vs4l_container_list *c)
{
	int ret = 0;
	struct vb_queue *q, *inq, *otq;

	inq = &queue->inqueue;
	otq = &queue->otqueue;

	if (c->direction == VS4L_DIRECTION_IN)
		q = inq;
	else
		q = otq;

	ret = vb_queue_unprepare(q, c);
	if (ret) {
		npu_err("vb_queue_prepare is fail(%d)\n", ret);
		goto p_err;
	}
p_err:
	return ret;
}

int npu_queue_dqbuf(struct npu_queue *queue, struct vs4l_container_list *c, bool nonblocking)
{
	int ret = 0;
	struct vb_queue *q;
	struct vb_bundle *bundle;

	BUG_ON(!queue);
	if (c->direction == VS4L_DIRECTION_IN)
		q = &queue->inqueue;
	else
		q = &queue->otqueue;

	ret = vb_queue_dqbuf(q, c, nonblocking);
	if (ret) {
		if (ret != -EWOULDBLOCK)
			npu_err("fail(%d) in vb_queue_dqbuf\n", ret);
		goto p_err;
	}

	if (c->index >= NPU_MAX_BUFFER) {
		npu_err("invalid in container index(%d)\n", c->index);
		ret = -EINVAL;
		goto p_err;
	}

	bundle = q->bufs[c->index];
	if (!bundle) {
		npu_err("null in bundle(%d)\n", c->index);
		ret = -EINVAL;
		goto p_err;
	}

	if (bundle->clist.index != c->index) {
		npu_err("not matched(%d != %d) in index\n", bundle->clist.index, c->index);
		ret = -EINVAL;
		goto p_err;
	}

	ret = CALL_QOPS(queue, deque, &bundle->clist);
	if (ret) {
		npu_err("fail(%d) in CALL_QOPS(deque)\n", ret);
		goto p_err;
	}
p_err:
	return ret;
}

void npu_queue_done(struct npu_queue *queue,
	struct vb_container_list *incl,
	struct vb_container_list *otcl,
	unsigned long flags)
{
	struct vb_queue *inq, *otq;
	struct vb_bundle *invb, *otvb;
	u32 count;

	BUG_ON(!queue);
	BUG_ON(!incl);
	BUG_ON(!otcl);

	inq = &queue->inqueue;
	otq = &queue->otqueue;

	count = (u32)atomic_read(&inq->process_count);
	if (list_empty(&inq->process_list)) {
		npu_err("empty in inqueue\n");
		BUG();
	}

	if (list_empty(&otq->process_list)) {
		npu_err("empty in otqueue\n");
		BUG();
	}

	invb = container_of(incl, struct vb_bundle, clist);
	otvb = container_of(otcl, struct vb_bundle, clist);

	if (invb->state != VB_BUF_STATE_PROCESS) {
		npu_err("invalid in invb state(%d)\n", invb->state);
		BUG();
	}

	if (otvb->state != VB_BUF_STATE_PROCESS) {
		npu_err("invalid in otvb state(%d)\n", otvb->state);
		BUG();
	}

	otvb->flags |= flags;
	vb_queue_done(otq, otvb);

	invb->flags |= flags;
	vb_queue_done(inq, invb);
}

void __npu_queue_print(struct vb_queue *q)
{
	int qcnt = atomic_read(&q->queued_count);
	int pcnt = atomic_read(&q->process_count);
	int dcnt = atomic_read(&q->done_count);
	u32 dir = q->direction;
	struct list_head *head;
	struct list_head *pos;

	npu_info("[npuQ] qcnt(%d), pcnt(%d), dcnt(%d)\n", qcnt, pcnt, dcnt);
	npu_info("[npuQ] dir(0x%x), &queued_list(0x%pK), &process_list(0x%pK), &done_list(0x%pK)\n",
			dir, &q->queued_list, &q->process_list, &q->done_list);

	head = &q->queued_list;

	for (pos = head->next ; pos != head ; pos = pos->next) {
		npu_info("[npuQ] pos(0x%pK)\n", pos);
		npu_info("[npuQ] pos->prev(0x%pK)\n", pos->prev);
		npu_info("[npuQ] pos->next(0x%pK)\n", pos->next);
	}

	head = &q->process_list;

	for (pos = head->next ; pos != head ; pos = pos->next) {
		npu_info("[npuQ] pos(0x%pK)\n", pos);
		npu_info("[npuQ] pos->prev(0x%pK)\n", pos->prev);
		npu_info("[npuQ] pos->next(0x%pK)\n", pos->next);
	}

	head = &q->done_list;

	for (pos = head->next ; pos != head ; pos = pos->next) {
		npu_info("[npuQ] pos(0x%pK)\n", pos);
		npu_info("[npuQ] pos->prev(0x%pK)\n", pos->prev);
		npu_info("[npuQ] pos->next(0x%pK)\n", pos->next);
	}
	return;
}