blob: 819b748dc02ed1c8c588cf2783ab53ec51122dea [file] [log] [blame]
Mark Kettenis50333c92022-01-22 20:38:18 +01001// SPDX-License-Identifier: GPL-2.0+
2/*
3 * (C) Copyright 2021 Mark Kettenis <kettenis@openbsd.org>
4 */
5
6#include <common.h>
7#include <dm.h>
8#include <mailbox.h>
9#include <mapmem.h>
10#include "nvme.h"
11#include <reset.h>
12
13#include <asm/io.h>
14#include <asm/arch/rtkit.h>
Janne Grunaue44d59c2022-06-14 09:09:07 +020015#include <asm/arch/sart.h>
Mark Kettenis50333c92022-01-22 20:38:18 +010016#include <linux/iopoll.h>
17
18/* ASC registers */
19#define REG_CPU_CTRL 0x0044
20#define REG_CPU_CTRL_RUN BIT(4)
21
22/* Apple NVMe registers */
23#define ANS_MAX_PEND_CMDS_CTRL 0x01210
24#define ANS_MAX_QUEUE_DEPTH 64
25#define ANS_BOOT_STATUS 0x01300
26#define ANS_BOOT_STATUS_OK 0xde71ce55
27#define ANS_MODESEL 0x01304
28#define ANS_UNKNOWN_CTRL 0x24008
29#define ANS_PRP_NULL_CHECK (1 << 11)
30#define ANS_LINEAR_SQ_CTRL 0x24908
31#define ANS_LINEAR_SQ_CTRL_EN (1 << 0)
32#define ANS_ASQ_DB 0x2490c
33#define ANS_IOSQ_DB 0x24910
34#define ANS_NVMMU_NUM 0x28100
35#define ANS_NVMMU_BASE_ASQ 0x28108
36#define ANS_NVMMU_BASE_IOSQ 0x28110
37#define ANS_NVMMU_TCB_INVAL 0x28118
38#define ANS_NVMMU_TCB_STAT 0x28120
39
40#define ANS_NVMMU_TCB_SIZE 0x4000
41#define ANS_NVMMU_TCB_PITCH 0x80
42
43/*
44 * The Apple NVMe controller includes an IOMMU known as NVMMU. The
45 * NVMMU is programmed through an array of TCBs. These TCBs are paired
46 * with the corresponding slot in the submission queues and need to be
47 * configured with the command details before a command is allowed to
48 * execute. This is necessary even for commands that don't do DMA.
49 */
50struct ans_nvmmu_tcb {
51 u8 opcode;
52 u8 flags;
53 u8 slot;
54 u8 pad0;
55 u32 prpl_len;
56 u8 pad1[16];
57 u64 prp1;
58 u64 prp2;
59};
60
61#define ANS_NVMMU_TCB_WRITE BIT(0)
62#define ANS_NVMMU_TCB_READ BIT(1)
63
64struct apple_nvme_priv {
65 struct nvme_dev ndev;
66 void *base; /* NVMe registers */
67 void *asc; /* ASC registers */
68 struct reset_ctl_bulk resets; /* ASC reset */
69 struct mbox_chan chan;
Janne Grunaue44d59c2022-06-14 09:09:07 +020070 struct apple_sart *sart;
71 struct apple_rtkit *rtk;
Mark Kettenis50333c92022-01-22 20:38:18 +010072 struct ans_nvmmu_tcb *tcbs[NVME_Q_NUM]; /* Submission queue TCBs */
73 u32 __iomem *q_db[NVME_Q_NUM]; /* Submission queue doorbell */
74};
75
76static int apple_nvme_setup_queue(struct nvme_queue *nvmeq)
77{
78 struct apple_nvme_priv *priv =
79 container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
80 struct nvme_dev *dev = nvmeq->dev;
81
82 switch (nvmeq->qid) {
83 case NVME_ADMIN_Q:
84 case NVME_IO_Q:
85 break;
86 default:
87 return -EINVAL;
88 }
89
90 priv->tcbs[nvmeq->qid] = (void *)memalign(4096, ANS_NVMMU_TCB_SIZE);
91 memset((void *)priv->tcbs[nvmeq->qid], 0, ANS_NVMMU_TCB_SIZE);
92
93 switch (nvmeq->qid) {
94 case NVME_ADMIN_Q:
95 priv->q_db[nvmeq->qid] =
96 ((void __iomem *)dev->bar) + ANS_ASQ_DB;
97 nvme_writeq((ulong)priv->tcbs[nvmeq->qid],
98 ((void __iomem *)dev->bar) + ANS_NVMMU_BASE_ASQ);
99 break;
100 case NVME_IO_Q:
101 priv->q_db[nvmeq->qid] =
102 ((void __iomem *)dev->bar) + ANS_IOSQ_DB;
103 nvme_writeq((ulong)priv->tcbs[nvmeq->qid],
104 ((void __iomem *)dev->bar) + ANS_NVMMU_BASE_IOSQ);
105 break;
106 }
107
108 return 0;
109}
110
111static void apple_nvme_submit_cmd(struct nvme_queue *nvmeq,
112 struct nvme_command *cmd)
113{
114 struct apple_nvme_priv *priv =
115 container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
116 struct ans_nvmmu_tcb *tcb;
117 u16 tail = nvmeq->sq_tail;
118
119 tcb = ((void *)priv->tcbs[nvmeq->qid]) + tail * ANS_NVMMU_TCB_PITCH;
120 memset(tcb, 0, sizeof(*tcb));
121 tcb->opcode = cmd->common.opcode;
122 tcb->flags = ANS_NVMMU_TCB_WRITE | ANS_NVMMU_TCB_READ;
123 tcb->slot = tail;
124 tcb->prpl_len = cmd->rw.length;
125 tcb->prp1 = cmd->common.prp1;
126 tcb->prp2 = cmd->common.prp2;
127
128 writel(tail, priv->q_db[nvmeq->qid]);
129}
130
131static void apple_nvme_complete_cmd(struct nvme_queue *nvmeq,
132 struct nvme_command *cmd)
133{
134 struct apple_nvme_priv *priv =
135 container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
136 struct ans_nvmmu_tcb *tcb;
137 u16 tail = nvmeq->sq_tail;
138
139 tcb = ((void *)priv->tcbs[nvmeq->qid]) + tail * ANS_NVMMU_TCB_PITCH;
140 memset(tcb, 0, sizeof(*tcb));
141 writel(tail, ((void __iomem *)nvmeq->dev->bar) + ANS_NVMMU_TCB_INVAL);
142 readl(((void __iomem *)nvmeq->dev->bar) + ANS_NVMMU_TCB_STAT);
143
144 if (++tail == nvmeq->q_depth)
145 tail = 0;
146 nvmeq->sq_tail = tail;
147}
148
Janne Grunaue44d59c2022-06-14 09:09:07 +0200149static int nvme_shmem_setup(void *cookie, struct apple_rtkit_buffer *buf)
150{
151 struct apple_nvme_priv *priv = (struct apple_nvme_priv *)cookie;
152
153 if (!buf || buf->dva || !buf->size)
154 return -1;
155
156 buf->buffer = memalign(SZ_16K, ALIGN(buf->size, SZ_16K));
157 if (!buf->buffer)
158 return -ENOMEM;
159
160 if (!sart_add_allowed_region(priv->sart, buf->buffer, buf->size)) {
161 free(buf->buffer);
162 buf->buffer = NULL;
163 buf->size = 0;
164 return -1;
165 }
166
167 buf->dva = (u64)buf->buffer;
168
169 return 0;
170}
171
172static void nvme_shmem_destroy(void *cookie, struct apple_rtkit_buffer *buf)
173{
174 struct apple_nvme_priv *priv = (struct apple_nvme_priv *)cookie;
175
176 if (!buf)
177 return;
178
179 if (buf->buffer) {
180 sart_remove_allowed_region(priv->sart, buf->buffer, buf->size);
181 free(buf->buffer);
182 buf->buffer = NULL;
183 buf->size = 0;
184 buf->dva = 0;
185 }
186}
187
Mark Kettenis50333c92022-01-22 20:38:18 +0100188static int apple_nvme_probe(struct udevice *dev)
189{
190 struct apple_nvme_priv *priv = dev_get_priv(dev);
191 fdt_addr_t addr;
Janne Grunaue44d59c2022-06-14 09:09:07 +0200192 ofnode of_sart;
193 u32 ctrl, stat, phandle;
Mark Kettenis50333c92022-01-22 20:38:18 +0100194 int ret;
195
196 priv->base = dev_read_addr_ptr(dev);
197 if (!priv->base)
198 return -EINVAL;
199
200 addr = dev_read_addr_index(dev, 1);
201 if (addr == FDT_ADDR_T_NONE)
202 return -EINVAL;
203 priv->asc = map_sysmem(addr, 0);
204
205 ret = reset_get_bulk(dev, &priv->resets);
206 if (ret < 0)
207 return ret;
208
209 ret = mbox_get_by_index(dev, 0, &priv->chan);
210 if (ret < 0)
211 return ret;
212
Janne Grunaue44d59c2022-06-14 09:09:07 +0200213 ret = dev_read_u32(dev, "apple,sart", &phandle);
214 if (ret < 0)
215 return ret;
216
217 of_sart = ofnode_get_by_phandle(phandle);
218 priv->sart = sart_init(of_sart);
219 if (!priv->sart)
220 return -EINVAL;
221
Mark Kettenis50333c92022-01-22 20:38:18 +0100222 ctrl = readl(priv->asc + REG_CPU_CTRL);
223 writel(ctrl | REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL);
224
Janne Grunaue44d59c2022-06-14 09:09:07 +0200225 priv->rtk = apple_rtkit_init(&priv->chan, priv, nvme_shmem_setup, nvme_shmem_destroy);
226 if (!priv->rtk)
227 return -ENOMEM;
228
229 ret = apple_rtkit_boot(priv->rtk);
230 if (ret < 0) {
231 printf("%s: NVMe apple_rtkit_boot returned: %d\n", __func__, ret);
Mark Kettenis50333c92022-01-22 20:38:18 +0100232 return ret;
Janne Grunaue44d59c2022-06-14 09:09:07 +0200233 }
Mark Kettenis50333c92022-01-22 20:38:18 +0100234
235 ret = readl_poll_sleep_timeout(priv->base + ANS_BOOT_STATUS, stat,
236 (stat == ANS_BOOT_STATUS_OK), 100,
237 500000);
238 if (ret < 0) {
239 printf("%s: NVMe firmware didn't boot\n", __func__);
240 return -ETIMEDOUT;
241 }
242
243 writel(ANS_LINEAR_SQ_CTRL_EN, priv->base + ANS_LINEAR_SQ_CTRL);
244 writel(((ANS_MAX_QUEUE_DEPTH << 16) | ANS_MAX_QUEUE_DEPTH),
245 priv->base + ANS_MAX_PEND_CMDS_CTRL);
246
247 writel(readl(priv->base + ANS_UNKNOWN_CTRL) & ~ANS_PRP_NULL_CHECK,
248 priv->base + ANS_UNKNOWN_CTRL);
249
250 strcpy(priv->ndev.vendor, "Apple");
251
252 writel((ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH) - 1,
253 priv->base + ANS_NVMMU_NUM);
254 writel(0, priv->base + ANS_MODESEL);
255
256 priv->ndev.bar = priv->base;
257 return nvme_init(dev);
258}
259
260static int apple_nvme_remove(struct udevice *dev)
261{
262 struct apple_nvme_priv *priv = dev_get_priv(dev);
263 u32 ctrl;
264
265 nvme_shutdown(dev);
266
Janne Grunaue44d59c2022-06-14 09:09:07 +0200267 apple_rtkit_shutdown(priv->rtk, APPLE_RTKIT_PWR_STATE_SLEEP);
Mark Kettenis50333c92022-01-22 20:38:18 +0100268
269 ctrl = readl(priv->asc + REG_CPU_CTRL);
270 writel(ctrl & ~REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL);
271
Janne Grunaue44d59c2022-06-14 09:09:07 +0200272 apple_rtkit_free(priv->rtk);
273 priv->rtk = NULL;
274
275 sart_free(priv->sart);
276 priv->sart = NULL;
277
Mark Kettenis50333c92022-01-22 20:38:18 +0100278 reset_assert_bulk(&priv->resets);
279 reset_deassert_bulk(&priv->resets);
280
281 return 0;
282}
283
284static const struct nvme_ops apple_nvme_ops = {
285 .setup_queue = apple_nvme_setup_queue,
286 .submit_cmd = apple_nvme_submit_cmd,
287 .complete_cmd = apple_nvme_complete_cmd,
288};
289
290static const struct udevice_id apple_nvme_ids[] = {
291 { .compatible = "apple,nvme-ans2" },
292 { /* sentinel */ }
293};
294
295U_BOOT_DRIVER(apple_nvme) = {
296 .name = "apple_nvme",
297 .id = UCLASS_NVME,
298 .of_match = apple_nvme_ids,
299 .priv_auto = sizeof(struct apple_nvme_priv),
300 .probe = apple_nvme_probe,
301 .remove = apple_nvme_remove,
302 .ops = &apple_nvme_ops,
303 .flags = DM_FLAG_OS_PREPARE,
304};