工作队列调用 aio_complete 时出现内核错误

Question

简版问题：

通过 io_submit 处理提交 IO（文件用 O_DIRECT 打开）。当在 fs address_space_operations .direct_IO 中调用 kiocb->ki_complete(aio_complete_rw) 时，一切正常。但是在workqueue工作函数中调用kiocb->ki_complete(aio_complete_rw)时（在fs.direct_IO的末尾排队），会发生kernel Oops.

我已经修改了 fs 的代码以接收来自 libaio 接口的请求，fs 会将请求排队到 workqueue 和 return EIOCBQUEUED。然后排队的工作只会调用 iocb->ki_complete，然后内核 Oops 发生。

[  161.904957] BUG: unable to handle kernel NULL pointer dereference at 0000000000000008
[  161.905850] IP: async_read_work+0x246/0x2e0 
[  161.906425] PGD 0 P4D 0 
[  161.906693] Oops: 0000 [#1] SMP NOPTI
[  161.910190] CPU: 0 PID: 208 Comm: kworker/0:3 Tainted: G           OE    4.15.7 #1
[  161.912829] RIP: 0010:async_read_work+0x246/0x2e0
[  161.913454] RSP: 0018:ffffb3aac0657e28 EFLAGS: 00010282
[  161.913980] RAX: 000000000000000c RBX: ffff9bd0f682c720 RCX: 0000000000000006
[  161.914728] RDX: 0000000000000000 RSI: 0000000000000086 RDI: ffff9bd0ffc16650
[  161.915481] RBP: ffff9bd0f689cc00 R08: 0000000000000001 R09: 000000000000024f
[  161.916196] R10: ffffda7300da0b00 R11: 000000000000024f R12: 0000000000000002
[  161.916959] R13: 0000000000000001 R14: ffff9bd0f689cc00 R15: 0000000000000001
[  161.917720] FS:  0000000000000000(0000) GS:ffff9bd0ffc00000(0000) knlGS:0000000000000000
[  161.918566] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  161.919141] CR2: 0000000000000008 CR3: 000000002820a005 CR4: 00000000003606f0
[  161.919896] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  161.920648] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  161.921362] Call Trace:
[  161.921697]  process_one_work+0x147/0x3d0
[  161.922119]  worker_thread+0x4a/0x440
[  161.922540]  kthread+0xf8/0x130
[  161.922864]  ? process_one_work+0x3d0/0x3d0
[  161.923308]  ? kthread_associate_blkcg+0x90/0x90
[  161.923827]  ret_from_fork+0x35/0x40
[  161.924193] Code: 8b b8 58 04 00 00 48 83 c7 70 e8 d6 ee c6 e3 4c 89 f7 e8 9e e0 db e3 48 89 df e8 96 e0 db e3 48 c7 c7 f3 af 46 c0 e8 d4 e7 c7 e3 <48> 8b 3c 25 08 00 00 00 31 d2 be 64 00 00 00 48 8b 47 10 e8 62 
[  161.926145] RIP: async_read_work+0x246/0x2e0  RSP: ffffb3aac0657e28
[  161.926911] CR2: 0000000000000008
[  161.927257] ---[ end trace 1f9420e2f5786b43 ]---

我试过在 fs 中调用 iocb->ki_complete（在 return EIOCBQUEUED 之前）并且效果很好。

PS:我正在尝试使用工作队列异步执行memcpy请求，memcpy请求是通过libaio接口传递的，并使用libaio通知机制。

代码如下：

struct async_work_stuct{
    struct iov_iter *a_iter;
    struct kiocb *a_iocb;
    int w_nr_segs;
    struct pinned_page *p;
    struct work_struct awork;
    struct task_struct *tsk;
};

struct pinned_page{
    struct page *pages[64];
    int num;
    int mapped;
};

排队工作 return EIOCBQUEUED:

const struct address_space_operations aops_xip = {
  .direct_IO= my_direct_IO,
};
static ssize_t my_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
{
...
unsigned long nr_segs = iter->nr_segs;
struct pinned_page *pp = (struct pinned_page*)kzalloc(nr_segs*sizeof(struct pinned_page),GFP_KERNEL);
struct async_work_stuct *read_work = (struct async_work_stuct*)kzalloc(sizeof(struct async_work_stuct),GFP_KERNEL);

read_work->w_nr_segs = nr_segs;
read_work->p = pp;
read_work->a_iter = iter;
read_work->a_iocb = iocb;
read_work->tsk = current;

INIT_WORK(&(read_work->awork), async_read_work);

bak = queue_work(sb->s_dio_done_wq, &(read_work->awork));

ret = -EIOCBQUEUED; 
return ret;
}

工作队列工作函数：

void async_read_work(struct work_struct *p_work)
{
    struct async_work_stuct *a_work = container_of(p_work, struct async_work_stuct, awork);
    struct pinned_page *pp = a_work->p;
    ...
    kfree(pp);
    pp = NULL;
    kfree(a_work);
    a_work = NULL;
    a_work->a_iocb->ki_complete(a_work->a_iocb,100, 0);
}

在我的工作函数中，我打印了 aio 完成函数的添加，然后内核 oops 发生了。

printk("aio_complete_addr = %u",a_work->a_iocb->ki_complete);

是不是说iocb初始化错误？我的用户进程通过io_submit提交IO，here aio complete函数指向aio_complete_rw.

Answer 1

在您的 async_read_work 函数中，您调用 kfree(a_work);、设置 a_work = NULL;，然后在此处取消引用 a_work：a_work->a_iocb->ki_complete(a_work->a_iocb,100, 0);.

可能的解决方案：

重新排列代码：

    a_work->a_iocb->ki_complete(a_work->a_iocb,100, 0);
    kfree(a_work);
    a_work = NULL; /* not really necessary */

使用局部变量：

    struct kiocb *iocb;
    ...
    iocb = a_work->a_iocb;
    kfree(a_work);
    a_work = NULL; /* not really necessary */
    iocb->ki_complete(iocb,100, 0);

工作队列调用 aio_complete 时出现内核错误

Kernel Oops when calling aio_complete by workqueue

c

aio

linux-device-driver

linux-kernel

workqueue