I found a way to be non-blocking on flock using inotify:

In case where we can't acquire the flock immediately, we fallback
registering an inotify watch over the filesystem to retry later.

IMHO the only hack here is the way I switch from a fd to the pathname.
But I'm not sure we wan't more arguments, or a more complexe approach.

Signed-off-by: Willow Barraco <contact@willowbarraco.fr>
---
 cmd/inotify/main.ha  | 27 +++++++++++++++++++++++++++
 ev/+linux/file.ha    |  4 ++++
 ev/+linux/inotify.ha | 30 ++++++++++++++++++++++++++++++
 ev/+linux/loop.ha    |  2 ++
 4 files changed, 63 insertions(+)
 create mode 100644 cmd/inotify/main.ha
 create mode 100644 ev/+linux/inotify.ha

diff --git a/cmd/inotify/main.ha b/cmd/inotify/main.ha
new file mode 100644
index 0000000..9fbc1cf
--- /dev/null
+++ b/cmd/inotify/main.ha
@@ -0,0 +1,27 @@
+use ev;
+use log;
+use rt;
+use os;
+use io;
+use fs;
+
+export fn main() void = {
+	const loop = ev::newloop()!;
+	defer ev::finish(&loop);
+
+	// touch the file
+	let writeme = os::create("writeme", fs::mode::USER_RW)!;
+	io::close(writeme)!;
+
+	let inotifyfile = ev::newinotify(&loop, &inotifycb)!;
+	defer ev::close(inotifyfile);
+
+	let watchfile = rt::inotify_add_watch(inotifyfile.fd, "writeme",
+		rt::INCLOSEWRITE)!;
+
+	for (ev::dispatch(&loop, -1)!) void;
+};
+
+fn inotifycb(file: *ev::file, event: rt::inotify_event) void = {
+	log::println("event:", event.mask);
+};
diff --git a/ev/+linux/file.ha b/ev/+linux/file.ha
index 73204fe..4b547a8 100644
--- a/ev/+linux/file.ha
+++ b/ev/+linux/file.ha
@@ -20,6 +20,7 @@ export type op = enum u64 {
 	RECVFROM    = 8 << 16,
 	SEND        = 9 << 16,
 	RECV        = 10 << 16,
+	INOTIFY     = 11 << 16,
 };
 
 export type fflags = enum uint {
@@ -157,6 +158,9 @@ fn file_epoll_ctl(file: *file) void = {
 		events |= rt::EPOLLIN;
 	case op::SEND, op::SENDTO =>
 		events |= rt::EPOLLOUT;
+	case op::INOTIFY =>
+		events &= ~rt::EPOLLONESHOT;
+		events |= rt::EPOLLIN;
 	case op::RECV, op::RECVFROM =>
 		events |= rt::EPOLLIN;
 	case =>
diff --git a/ev/+linux/inotify.ha b/ev/+linux/inotify.ha
new file mode 100644
index 0000000..ced8dc5
--- /dev/null
+++ b/ev/+linux/inotify.ha
@@ -0,0 +1,30 @@
+use errors;
+use io;
+use rt;
+
+// A callback which executes when an inotify event is received.
+export type inotifycb = fn(file: *file, event: rt::inotify_event) void;
+
+// Creates a new inotify
+export fn newinotify(
+	loop: *loop,
+	cb: *inotifycb,
+) (*file | rt::errno | errors::error) = {
+	const fd = rt::inotify_init()?;
+	const file = register(loop, fd)?;
+	file.op = op::INOTIFY;
+	file.cb = cb;
+	file_epoll_ctl(file);
+	return file;
+};
+
+fn inotify_ready(file: *file, ev: *rt::epoll_event) void = {
+	assert(file.op == op::INOTIFY);
+
+	let buf = rt::inotify_event {...};
+	rt::read(file.fd, &buf, size(rt::inotify_event))!;
+
+	assert(file.cb != null);
+	const cb = file.cb: *inotifycb;
+	cb(file, buf);
+};
diff --git a/ev/+linux/loop.ha b/ev/+linux/loop.ha
index aec5cc0..5ffa9c4 100644
--- a/ev/+linux/loop.ha
+++ b/ev/+linux/loop.ha
@@ -129,6 +129,8 @@ export fn dispatch(
 			send_ready(file, ev);
 		case op::RECV =>
 			recv_ready(file, ev);
+		case op::INOTIFY =>
+			inotify_ready(file, ev);
 		case =>
 			assert(pending & ~(op::READV | op::WRITEV) == 0);
 		};
-- 
2.39.1

Signed-off-by: Willow Barraco <contact@willowbarraco.fr>
---
 cmd/flock/main.ha  | 40 ++++++++++++++++++++++
 ev/+linux/file.ha  | 11 ++++++
 ev/+linux/flock.ha | 84 ++++++++++++++++++++++++++++++++++++++++++++++
 ev/+linux/loop.ha  |  2 ++
 4 files changed, 137 insertions(+)
 create mode 100644 cmd/flock/main.ha
 create mode 100644 ev/+linux/flock.ha

diff --git a/cmd/flock/main.ha b/cmd/flock/main.ha
new file mode 100644
index 0000000..e50386e
--- /dev/null
+++ b/cmd/flock/main.ha
@@ -0,0 +1,40 @@
+use ev;
+use log;
+use time;
+use rt;
+use os;
+use io;
+use fs;
+use errors;
+
+export fn main() void = {
+	const loop = ev::newloop()!;
+	defer ev::finish(&loop);
+
+	const lockfile = os::create("lock", fs::mode::USER_RW)!;
+	defer io::close(lockfile)!;
+
+	const flockfile = ev::newflock(&loop, &lockedcb, lockfile, rt::LOCK_EX)!;
+
+	const locked = true;
+	ev::setuser(flockfile, &locked);
+
+	for (ev::dispatch(&loop, -1)!) {
+		if (!locked) {
+			log::println("file freed");
+			time::sleep(1 * time::SECOND);
+
+			const flockfile = ev::newflock(&loop, &lockedcb, lockfile, rt::LOCK_EX)!;
+			ev::setuser(flockfile, &locked);
+			locked = true;
+		};
+	};
+};
+
+fn lockedcb(file: *ev::file) void = {
+	log::println("file locked");
+	time::sleep(2 * time::SECOND);
+
+	const locked = ev::getuser(file): *bool;
+	*locked = false;
+};
diff --git a/ev/+linux/file.ha b/ev/+linux/file.ha
index 4b547a8..06ff2f8 100644
--- a/ev/+linux/file.ha
+++ b/ev/+linux/file.ha
@@ -21,6 +21,7 @@ export type op = enum u64 {
 	SEND        = 9 << 16,
 	RECV        = 10 << 16,
 	INOTIFY     = 11 << 16,
+	FLOCK       = 12 << 16,
 };
 
 export type fflags = enum uint {
@@ -53,6 +54,11 @@ export type file = struct {
 			dest: ip::addr,
 			port: u16,
 		},
+		struct {
+			lockfile: io::file,
+			pendingop: int,
+			pendingcb: *lockcb,
+		}
 	},
 };
 
@@ -103,6 +109,9 @@ export fn unregister(file: *file) void = {
 	if (file.op == op::SIGNAL) {
 		signal_restore(file);
 	};
+	if (file.op == op::FLOCK) {
+		rt::flock(file.lockfile, rt::LOCK_UN)!;
+	};
 	free(file);
 };
 
@@ -161,6 +170,8 @@ fn file_epoll_ctl(file: *file) void = {
 	case op::INOTIFY =>
 		events &= ~rt::EPOLLONESHOT;
 		events |= rt::EPOLLIN;
+	case op::FLOCK =>
+		events |= rt::EPOLLOUT;
 	case op::RECV, op::RECVFROM =>
 		events |= rt::EPOLLIN;
 	case =>
diff --git a/ev/+linux/flock.ha b/ev/+linux/flock.ha
new file mode 100644
index 0000000..ba53166
--- /dev/null
+++ b/ev/+linux/flock.ha
@@ -0,0 +1,84 @@
+use errors;
+use io;
+use rt;
+use os;
+use strconv;
+use path;
+
+// A callback which executes when a lock is acquired.
+export type lockcb = fn(file: *file) void;
+
+// Creates a new flock. Lock immediately if possible or register a filesystem
+// inotify watch to flock when possible.
+export fn newflock(
+	loop: *loop,
+	cb: *lockcb,
+	lockfile: io::file,
+	op: int,
+) (*file | rt::errno | errors::error) = {
+	match(rt::flock(lockfile, op | rt::LOCK_NB)) {
+	case let fd: int =>
+		const file = register(loop, fd)?;
+		file.op = op::FLOCK;
+		file.cb = cb;
+		file.lockfile = lockfile;
+		file_epoll_ctl(file);
+		return file;
+	case let err: rt::errno =>
+		if (err: int != rt::EWOULDBLOCK) {
+			return err;
+		};
+
+		const buf = path::init();
+		path::add(&buf, "/proc/self/fd/", strconv::integertos(lockfile: int))!;
+		const realpath = os::readlink(path::string(&buf))!;
+
+		// TODO: check realpath and return err in case fd isn't a fs file
+
+		const file = newinotify(loop, &flock_retry_lock)!;
+		rt::inotify_add_watch(file.fd, realpath, rt::INCLOSE)!;
+
+		file.lockfile = lockfile;
+		file.pendingop = op;
+		file.pendingcb = cb;
+		return file;
+	};
+};
+
+fn flock_retry_lock(file: *file, event: rt::inotify_event) void = {
+	match(rt::flock(file.lockfile, file.pendingop | rt::LOCK_NB)) {
+	case let fd: int =>
+		// We do things manually here to not re-allocate the file, to preserve
+		// the user ownership over it
+		rt::epoll_ctl(file.ev.fd, rt::EPOLL_CTL_DEL, file.fd, null)!;
+		io::close(file.fd)!;
+
+		let ev = rt::epoll_event {
+			events = 0,
+			...
+		};
+		ev.data.ptr = file;
+		rt::epoll_ctl(file.ev.fd, rt::EPOLL_CTL_ADD, fd, &ev)!;
+
+		file.fd = fd;
+		file.op = op::FLOCK;
+		file.cb = file.pendingcb;
+		file.lockfile = file.lockfile;
+		file.user = file.user;
+		file_epoll_ctl(file);
+	case rt::errno =>
+		// TODO handle non EWOULDBLOCK errors ?
+		yield;
+	};
+};
+
+fn flock_ready(file: *file, ev: *rt::epoll_event) void = {
+	assert(file.op == op::FLOCK);
+
+	assert(file.cb != null);
+	const cb = file.cb: *lockcb;
+	cb(file);
+
+	rt::flock(file.lockfile, rt::LOCK_UN)!;
+	unregister(file);
+};
diff --git a/ev/+linux/loop.ha b/ev/+linux/loop.ha
index 5ffa9c4..d0bd0fa 100644
--- a/ev/+linux/loop.ha
+++ b/ev/+linux/loop.ha
@@ -131,6 +131,8 @@ export fn dispatch(
 			recv_ready(file, ev);
 		case op::INOTIFY =>
 			inotify_ready(file, ev);
+		case op::FLOCK =>
+			flock_ready(file, ev);
 		case =>
 			assert(pending & ~(op::READV | op::WRITEV) == 0);
 		};
-- 
2.39.1

This flock fallback doesn't even works. That was actually busybox flock
that was closing the file, triggering this inotifywait event. It looks
like flock doesn't signal any event.

Man I am not thrilled by the magic inotify fallback case here

> Man I am not thrilled by the magic inotify fallback case here

Nah and worse, it doesn't even works as expected. The flock shell
command trigger a inotify event but that isn't something expected
anyway.