⛏️ index : haiku.git

Diff

 src/add-ons/kernel/drivers/disk/Jamfile            |   5 +++--
 src/add-ons/kernel/drivers/disk/nvme/Jamfile       |  23 +++++++++++++++++++++++
 src/add-ons/kernel/drivers/disk/nvme/nvme_disk.cpp | 554 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 580 insertions(+), 2 deletions(-)

diff --git a/src/add-ons/kernel/drivers/disk/Jamfile b/src/add-ons/kernel/drivers/disk/Jamfile
index 9dfc902..6aa2de8 100644
--- a/src/add-ons/kernel/drivers/disk/Jamfile
+++ b/src/add-ons/kernel/drivers/disk/Jamfile
@@ -1,8 +1,9 @@
SubDir HAIKU_TOP src add-ons kernel drivers disk ;

SubInclude HAIKU_TOP src add-ons kernel drivers disk floppy ;
SubInclude HAIKU_TOP src add-ons kernel drivers disk norflash ;
SubInclude HAIKU_TOP src add-ons kernel drivers disk mmc ;
SubInclude HAIKU_TOP src add-ons kernel drivers disk norflash ;
SubInclude HAIKU_TOP src add-ons kernel drivers disk nvme ;
SubInclude HAIKU_TOP src add-ons kernel drivers disk scsi ;
SubInclude HAIKU_TOP src add-ons kernel drivers disk usb ;
SubInclude HAIKU_TOP src add-ons kernel drivers disk virtual ;
SubInclude HAIKU_TOP src add-ons kernel drivers disk virtual ;
diff --git a/src/add-ons/kernel/drivers/disk/nvme/Jamfile b/src/add-ons/kernel/drivers/disk/nvme/Jamfile
new file mode 100644
index 0000000..48286fb 100644
--- /dev/null
+++ b/src/add-ons/kernel/drivers/disk/nvme/Jamfile
@@ -1,0 +1,23 @@
SubDir HAIKU_TOP src add-ons kernel drivers disk nvme ;

UsePrivateKernelHeaders ;
SubDirHdrs $(HAIKU_TOP) src system kernel device_manager ;

UseHeaders [ FDirName $(SUBDIR) ] : true ;

SEARCH_SOURCE += [ FDirName $(SUBDIR) libnvme ] ;
SEARCH_SOURCE += [ FDirName $(SUBDIR) compat ] ;

KernelAddon nvme_disk :
	nvme_disk.cpp
	libnvme_haiku.cpp

	nvme.c
	nvme_admin.c
	nvme_common.c
	nvme_ctrlr.c
	nvme_ns.c
	nvme_qpair.c
	nvme_quirks.c
	nvme_request.c
;
diff --git a/src/add-ons/kernel/drivers/disk/nvme/nvme_disk.cpp b/src/add-ons/kernel/drivers/disk/nvme/nvme_disk.cpp
new file mode 100644
index 0000000..a55535e 100644
--- /dev/null
+++ b/src/add-ons/kernel/drivers/disk/nvme/nvme_disk.cpp
@@ -1,0 +1,554 @@
/*
 * Copyright 2019, Haiku, Inc. All rights reserved.
 * Distributed under the terms of the MIT License.
 *
 * Authors:
 *		Augustin Cavalier <waddlesplash>
 */


#include <stdio.h>
#include <stdlib.h>

#include <kernel.h>
#include <util/AutoLock.h>

#include <fs/devfs.h>
#include <bus/PCI.h>

extern "C" {
#include <libnvme/nvme.h>
}


#define TRACE_NVME_DISK
#ifdef TRACE_NVME_DISK
#	define TRACE(x...) dprintf("nvme_disk: " x)
#else
#	define TRACE(x...) ;
#endif
#define TRACE_ERROR(x...)	dprintf("\33[33mnvme_disk:\33[0m " x)
#define CALLED() 			TRACE("CALLED %s\n", __PRETTY_FUNCTION__)


static const uint8 kDriveIcon[] = {
	0x6e, 0x63, 0x69, 0x66, 0x08, 0x03, 0x01, 0x00, 0x00, 0x02, 0x00, 0x16,
	0x02, 0x3c, 0xc7, 0xee, 0x38, 0x9b, 0xc0, 0xba, 0x16, 0x57, 0x3e, 0x39,
	0xb0, 0x49, 0x77, 0xc8, 0x42, 0xad, 0xc7, 0x00, 0xff, 0xff, 0xd3, 0x02,
	0x00, 0x06, 0x02, 0x3c, 0x96, 0x32, 0x3a, 0x4d, 0x3f, 0xba, 0xfc, 0x01,
	0x3d, 0x5a, 0x97, 0x4b, 0x57, 0xa5, 0x49, 0x84, 0x4d, 0x00, 0x47, 0x47,
	0x47, 0xff, 0xa5, 0xa0, 0xa0, 0x02, 0x00, 0x16, 0x02, 0xbc, 0x59, 0x2f,
	0xbb, 0x29, 0xa7, 0x3c, 0x0c, 0xe4, 0xbd, 0x0b, 0x7c, 0x48, 0x92, 0xc0,
	0x4b, 0x79, 0x66, 0x00, 0x7d, 0xff, 0xd4, 0x02, 0x00, 0x06, 0x02, 0x38,
	0xdb, 0xb4, 0x39, 0x97, 0x33, 0xbc, 0x4a, 0x33, 0x3b, 0xa5, 0x42, 0x48,
	0x6e, 0x66, 0x49, 0xee, 0x7b, 0x00, 0x59, 0x67, 0x56, 0xff, 0xeb, 0xb2,
	0xb2, 0x03, 0xa7, 0xff, 0x00, 0x03, 0xff, 0x00, 0x00, 0x04, 0x01, 0x80,
	0x07, 0x0a, 0x06, 0x22, 0x3c, 0x22, 0x49, 0x44, 0x5b, 0x5a, 0x3e, 0x5a,
	0x31, 0x39, 0x25, 0x0a, 0x04, 0x22, 0x3c, 0x44, 0x4b, 0x5a, 0x31, 0x39,
	0x25, 0x0a, 0x04, 0x44, 0x4b, 0x44, 0x5b, 0x5a, 0x3e, 0x5a, 0x31, 0x0a,
	0x04, 0x22, 0x3c, 0x22, 0x49, 0x44, 0x5b, 0x44, 0x4b, 0x08, 0x02, 0x27,
	0x43, 0xb8, 0x14, 0xc1, 0xf1, 0x08, 0x02, 0x26, 0x43, 0x29, 0x44, 0x0a,
	0x05, 0x44, 0x5d, 0x49, 0x5d, 0x60, 0x3e, 0x5a, 0x3b, 0x5b, 0x3f, 0x08,
	0x0a, 0x07, 0x01, 0x06, 0x00, 0x0a, 0x00, 0x01, 0x00, 0x10, 0x01, 0x17,
	0x84, 0x00, 0x04, 0x0a, 0x01, 0x01, 0x01, 0x00, 0x0a, 0x02, 0x01, 0x02,
	0x00, 0x0a, 0x03, 0x01, 0x03, 0x00, 0x0a, 0x04, 0x01, 0x04, 0x10, 0x01,
	0x17, 0x85, 0x20, 0x04, 0x0a, 0x06, 0x01, 0x05, 0x30, 0x24, 0xb3, 0x99,
	0x01, 0x17, 0x82, 0x00, 0x04, 0x0a, 0x05, 0x01, 0x05, 0x30, 0x20, 0xb2,
	0xe6, 0x01, 0x17, 0x82, 0x00, 0x04
};


#define NVME_DISK_DRIVER_MODULE_NAME 	"drivers/disk/nvme_disk/driver_v1"
#define NVME_DISK_DEVICE_MODULE_NAME 	"drivers/disk/nvme_disk/device_v1"
#define NVME_DISK_DEVICE_ID_GENERATOR	"nvme_disk/device_id"


static device_manager_info* sDeviceManager;

typedef struct {
	device_node*			node;
	pci_device_module_info*	pci;
	pci_device*				device;
	pci_info				info;

	struct nvme_ctrlr*		ctrlr;
	struct nvme_ns*			ns;

	uint64					capacity;
	uint32					block_size;
	status_t				media_status;

	struct nvme_qpair*		qpair;
	mutex					qpair_mtx;
} nvme_disk_driver_info;


typedef struct {
	nvme_disk_driver_info*		info;
} nvme_disk_handle;


static status_t
get_geometry(nvme_disk_handle* handle, device_geometry* geometry)
{
	nvme_disk_driver_info* info = handle->info;

	devfs_compute_geometry_size(geometry, info->capacity, info->block_size);

	geometry->device_type = B_DISK;
	geometry->removable = false;

	geometry->read_only = true; /* TODO: Write support! */
	geometry->write_once = false;

	TRACE("get_geometry(): %" B_PRId32 ", %" B_PRId32 ", %" B_PRId32 ", %" B_PRId32 ", %d, %d, %d, %d\n",
		geometry->bytes_per_sector, geometry->sectors_per_track,
		geometry->cylinder_count, geometry->head_count, geometry->device_type,
		geometry->removable, geometry->read_only, geometry->write_once);

	return B_OK;
}


static int
log2(uint32 x)
{
	int y;

	for (y = 31; y >= 0; --y) {
		if (x == ((uint32)1 << y))
			break;
	}

	return y;
}


static void
nvme_disk_set_capacity(nvme_disk_driver_info* info, uint64 capacity,
	uint32 blockSize)
{
	TRACE("set_capacity(device = %p, capacity = %" B_PRIu64 ", blockSize = %" B_PRIu32 ")\n",
		info, capacity, blockSize);

	// get log2, if possible
	uint32 blockShift = log2(blockSize);

	if ((1UL << blockShift) != blockSize)
		blockShift = 0;

	info->capacity = capacity;
	info->block_size = blockSize;
}


//	#pragma mark - device module API


static status_t
nvme_disk_init_device(void* _info, void** _cookie)
{
	CALLED();
	nvme_disk_driver_info* info = (nvme_disk_driver_info*)_info;

	device_node* parent = sDeviceManager->get_parent_node(info->node);
	device_node* pciParent = sDeviceManager->get_parent_node(parent);
	sDeviceManager->get_driver(pciParent, (driver_module_info**)&info->pci,
		(void**)&info->device);
	sDeviceManager->put_node(pciParent);
	sDeviceManager->put_node(parent);

	info->pci->get_pci_info(info->device, &info->info);

	// construct the libnvme pci_device struct
	pci_device* device = new pci_device;
	*device = {
		.vendor_id = info->info.vendor_id,
		.device_id = info->info.device_id,
		.subvendor_id = 0,
		.subdevice_id = 0,

		.domain = 0,
		.bus = info->info.bus,
		.dev = info->info.device,
		.func = info->info.function,

		.pci_info = &info->info,
	};

	// open the controller
	info->ctrlr = nvme_ctrlr_open(device, NULL);
	if (info->ctrlr == NULL) {
		TRACE_ERROR("failed to open the controller!\n");
		return B_ERROR;
	}

	struct nvme_ctrlr_stat cstat;
	int err = nvme_ctrlr_stat(info->ctrlr, &cstat);
	if (err != 0) {
		TRACE_ERROR("failed to get controller information!\n");
		return -err;
	}

	// TODO: export more than just the first namespace!
	info->ns = nvme_ns_open(info->ctrlr, cstat.ns_ids[0]);
	if (info->ns == NULL) {
		TRACE_ERROR("failed to open namespace!\n");
		return B_ERROR;
	}

	struct nvme_ns_stat nsstat;
	err = nvme_ns_stat(info->ns, &nsstat);
	if (err != 0) {
		TRACE_ERROR("failed to get namespace information!\n");
		return -err;
	}

	// store capacity information
	nvme_disk_set_capacity(info, nsstat.sectors, nsstat.sector_size);

	TRACE("capacity: %" B_PRIu64 ", block_size %" B_PRIu32 "\n",
		info->capacity, info->block_size);

	// allocate a qpair
	// TODO: allocate more than one qpair
	info->qpair = nvme_ioqp_get(info->ctrlr, (enum nvme_qprio)0, 0);
	if (info->qpair == NULL) {
		TRACE_ERROR("failed to allocate qpair!\n");
		return B_ERROR;
	}

	mutex_init(&info->qpair_mtx, "qpair mtx");

	*_cookie = info;
	return B_OK;
}


static void
nvme_disk_uninit_device(void* _cookie)
{
	CALLED();
	nvme_disk_driver_info* info = (nvme_disk_driver_info*)_cookie;
}


static status_t
nvme_disk_open(void* _info, const char* path, int openMode, void** _cookie)
{
	CALLED();

	nvme_disk_driver_info* info = (nvme_disk_driver_info*)_info;
	nvme_disk_handle* handle = (nvme_disk_handle*)malloc(
		sizeof(nvme_disk_handle));
	if (handle == NULL)
		return B_NO_MEMORY;

	handle->info = info;

	*_cookie = handle;
	return B_OK;
}


static status_t
nvme_disk_close(void* cookie)
{
	CALLED();

	nvme_disk_handle* handle = (nvme_disk_handle*)cookie;
	return B_OK;
}


static status_t
nvme_disk_free(void* cookie)
{
	CALLED();

	nvme_disk_handle* handle = (nvme_disk_handle*)cookie;
	free(handle);
	return B_OK;
}


static void
disk_read_callback(bool* done, const struct nvme_cpl*)
{
	*done = true;
}


static status_t
nvme_disk_read(void* cookie, off_t pos, void* buffer, size_t* _length)
{
	CALLED();
	nvme_disk_handle* handle = (nvme_disk_handle*)cookie;

	bool done = false;
	size_t rounded_len = ROUNDUP(*_length, handle->info->block_size);

	void* real_buffer;
	if (rounded_len == *_length && !IS_USER_ADDRESS(buffer))
		real_buffer = buffer;
	else
		real_buffer = malloc(rounded_len);

	MutexLocker _(handle->info->qpair_mtx);
	int ret = nvme_ns_read(handle->info->ns, handle->info->qpair,
		real_buffer, pos / handle->info->block_size, rounded_len / handle->info->block_size,
		(nvme_cmd_cb)disk_read_callback, &done, 0);
	if (ret != 0)
		return ret;

	while (!done) {
		nvme_ioqp_poll(handle->info->qpair, 1);
		snooze(1);
	}

	status_t status = B_OK;
	if (real_buffer != buffer) {
		if (IS_USER_ADDRESS(buffer))
			status = user_memcpy(buffer, real_buffer, *_length);
		else
			memcpy(buffer, real_buffer, *_length);
		free(real_buffer);
	}

	return status;
}


static status_t
nvme_disk_write(void* cookie, off_t pos, const void* buffer,
	size_t* _length)
{
	CALLED();
	nvme_disk_handle* handle = (nvme_disk_handle*)cookie;

	return B_NOT_SUPPORTED;
}


static status_t
nvme_disk_ioctl(void* cookie, uint32 op, void* buffer, size_t length)
{
	CALLED();
	nvme_disk_handle* handle = (nvme_disk_handle*)cookie;
	nvme_disk_driver_info* info = handle->info;

	TRACE("ioctl(op = %" B_PRId32 ")\n", op);

	switch (op) {
		case B_GET_MEDIA_STATUS:
		{
			*(status_t *)buffer = info->media_status;
			info->media_status = B_OK;
			return B_OK;
			break;
		}

		case B_GET_DEVICE_SIZE:
		{
			size_t size = info->capacity * info->block_size;
			return user_memcpy(buffer, &size, sizeof(size_t));
		}

		case B_GET_GEOMETRY:
		{
			if (buffer == NULL /*|| length != sizeof(device_geometry)*/)
				return B_BAD_VALUE;

		 	device_geometry geometry;
			status_t status = get_geometry(handle, &geometry);
			if (status != B_OK)
				return status;

			return user_memcpy(buffer, &geometry, sizeof(device_geometry));
		}

		case B_GET_ICON_NAME:
			return user_strlcpy((char*)buffer, "devices/drive-harddisk",
				B_FILE_NAME_LENGTH);

		case B_GET_VECTOR_ICON:
		{
			device_icon iconData;
			if (length != sizeof(device_icon))
				return B_BAD_VALUE;
			if (user_memcpy(&iconData, buffer, sizeof(device_icon)) != B_OK)
				return B_BAD_ADDRESS;

			if (iconData.icon_size >= (int32)sizeof(kDriveIcon)) {
				if (user_memcpy(iconData.icon_data, kDriveIcon,
						sizeof(kDriveIcon)) != B_OK)
					return B_BAD_ADDRESS;
			}

			iconData.icon_size = sizeof(kDriveIcon);
			return user_memcpy(buffer, &iconData, sizeof(device_icon));
		}

		/*case B_FLUSH_DRIVE_CACHE:
			return synchronize_cache(info);*/
	}

	return B_DEV_INVALID_IOCTL;
}


//	#pragma mark - driver module API


static float
nvme_disk_supports_device(device_node *parent)
{
	CALLED();

	const char* bus;
	uint16 baseClass, subClass;

	if (sDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false) != B_OK
		|| sDeviceManager->get_attr_uint16(parent, B_DEVICE_TYPE, &baseClass, false) != B_OK
		|| sDeviceManager->get_attr_uint16(parent, B_DEVICE_SUB_TYPE, &subClass, false) != B_OK)
		return -1.0f;

	TRACE("strcmp: %d, baseclass: %d\n", strcmp(bus, "pci") != 0, baseClass != PCI_mass_storage);

	if (strcmp(bus, "pci") != 0 || baseClass != PCI_mass_storage)
		return 0.0f;

	if (subClass != PCI_nvm)
		return 0.0f;

	TRACE("NVMe device found!\n");
	return 1.0f;
}


static status_t
nvme_disk_register_device(device_node *node)
{
	CALLED();

	// ready to register
	device_attr attrs[] = {
		{ NULL }
	};

	return sDeviceManager->register_node(node, NVME_DISK_DRIVER_MODULE_NAME,
		attrs, NULL, NULL);
}


static status_t
nvme_disk_init_driver(device_node* node, void** cookie)
{
	CALLED();

	int ret = nvme_lib_init((enum nvme_log_level)0, (enum nvme_log_facility)0, NULL);
	if (ret != 0) {
		TRACE_ERROR("libnvme initialization failed!\n");
		return ret;
	}

	nvme_disk_driver_info* info = (nvme_disk_driver_info*)malloc(
		sizeof(nvme_disk_driver_info));
	if (info == NULL)
		return B_NO_MEMORY;

	memset(info, 0, sizeof(*info));

	info->media_status = B_OK;
	info->node = node;

	*cookie = info;
	return B_OK;
}


static void
nvme_disk_uninit_driver(void* _cookie)
{
	CALLED();

	nvme_disk_driver_info* info = (nvme_disk_driver_info*)_cookie;
	free(info);
}


static status_t
nvme_disk_register_child_devices(void* _cookie)
{
	CALLED();

	nvme_disk_driver_info* info = (nvme_disk_driver_info*)_cookie;
	status_t status;

	int32 id = sDeviceManager->create_id(NVME_DISK_DEVICE_ID_GENERATOR);
	if (id < 0)
		return id;

	char name[64];
	snprintf(name, sizeof(name), "disk/nvme/%" B_PRId32 "/raw",
		id);

	status = sDeviceManager->publish_device(info->node, name,
		NVME_DISK_DEVICE_MODULE_NAME);

	return status;
}


//	#pragma mark -


module_dependency module_dependencies[] = {
	{B_DEVICE_MANAGER_MODULE_NAME, (module_info**)&sDeviceManager},
	{}
};

struct device_module_info sNvmeDiskDevice = {
	{
		NVME_DISK_DEVICE_MODULE_NAME,
		0,
		NULL
	},

	nvme_disk_init_device,
	nvme_disk_uninit_device,
	NULL, // remove,

	nvme_disk_open,
	nvme_disk_close,
	nvme_disk_free,
	nvme_disk_read,
	nvme_disk_write,
	NULL,
	nvme_disk_ioctl,

	NULL,	// select
	NULL,	// deselect
};

struct driver_module_info sNvmeDiskDriver = {
	{
		NVME_DISK_DRIVER_MODULE_NAME,
		0,
		NULL
	},

	nvme_disk_supports_device,
	nvme_disk_register_device,
	nvme_disk_init_driver,
	nvme_disk_uninit_driver,
	nvme_disk_register_child_devices,
	NULL,	// rescan
	NULL,	// removed
};

module_info* modules[] = {
	(module_info*)&sNvmeDiskDriver,
	(module_info*)&sNvmeDiskDevice,
	NULL
};