blob: 126a365716676a0eecb4b1e6cb97f7f8df1bce9a [file] [log] [blame]
Thomas Gleixnerc942fdd2019-05-27 08:55:06 +02001/* SPDX-License-Identifier: GPL-2.0-or-later */
Jérôme Glisse133ff0e2017-09-08 16:11:23 -07002/*
3 * Copyright 2013 Red Hat Inc.
4 *
Jérôme Glissef813f212018-10-30 15:04:06 -07005 * Authors: Jérôme Glisse <[email protected]>
Jérôme Glisse133ff0e2017-09-08 16:11:23 -07006 *
Mike Rapoportee657282022-06-27 09:00:26 +03007 * See Documentation/mm/hmm.rst for reasons and overview of what HMM is.
Jérôme Glisse133ff0e2017-09-08 16:11:23 -07008 */
9#ifndef LINUX_HMM_H
10#define LINUX_HMM_H
11
Christoph Hellwig730ff522022-02-16 15:31:35 +110012#include <linux/mm.h>
Jérôme Glisse133ff0e2017-09-08 16:11:23 -070013
Christoph Hellwig730ff522022-02-16 15:31:35 +110014struct mmu_interval_notifier;
Jérôme Glisse4ef589d2017-09-08 16:11:58 -070015
Jérôme Glisse133ff0e2017-09-08 16:11:23 -070016/*
Jason Gunthorpe2733ea12020-05-01 15:20:48 -030017 * On output:
18 * 0 - The page is faultable and a future call with
19 * HMM_PFN_REQ_FAULT could succeed.
20 * HMM_PFN_VALID - the pfn field points to a valid PFN. This PFN is at
21 * least readable. If dev_private_owner is !NULL then this could
22 * point at a DEVICE_PRIVATE page.
23 * HMM_PFN_WRITE - if the page memory can be written to (requires HMM_PFN_VALID)
24 * HMM_PFN_ERROR - accessing the pfn is impossible and the device should
25 * fail. ie poisoned memory, special pages, no vma, etc
Jérôme Glissef88a1e92018-04-10 16:29:06 -070026 *
Jason Gunthorpe2733ea12020-05-01 15:20:48 -030027 * On input:
28 * 0 - Return the current state of the page, do not fault it.
29 * HMM_PFN_REQ_FAULT - The output must have HMM_PFN_VALID or hmm_range_fault()
30 * will fail
31 * HMM_PFN_REQ_WRITE - The output must have HMM_PFN_WRITE or hmm_range_fault()
32 * will fail. Must be combined with HMM_PFN_REQ_FAULT.
Jérôme Glissef88a1e92018-04-10 16:29:06 -070033 */
Jason Gunthorpe2733ea12020-05-01 15:20:48 -030034enum hmm_pfn_flags {
Ralph Campbell3b50a6e2020-07-01 15:53:49 -070035 /* Output fields and flags */
Jason Gunthorpe2733ea12020-05-01 15:20:48 -030036 HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),
37 HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),
38 HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),
Ralph Campbell3b50a6e2020-07-01 15:53:49 -070039 HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),
Jason Gunthorpe2733ea12020-05-01 15:20:48 -030040
41 /* Input flags */
42 HMM_PFN_REQ_FAULT = HMM_PFN_VALID,
43 HMM_PFN_REQ_WRITE = HMM_PFN_WRITE,
44
Ralph Campbell3b50a6e2020-07-01 15:53:49 -070045 HMM_PFN_FLAGS = 0xFFUL << HMM_PFN_ORDER_SHIFT,
Jérôme Glissef88a1e92018-04-10 16:29:06 -070046};
47
48/*
Jason Gunthorpe2733ea12020-05-01 15:20:48 -030049 * hmm_pfn_to_page() - return struct page pointed to by a device entry
Jérôme Glissef88a1e92018-04-10 16:29:06 -070050 *
Jason Gunthorpe2733ea12020-05-01 15:20:48 -030051 * This must be called under the caller 'user_lock' after a successful
52 * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
53 * already.
Jérôme Glisse133ff0e2017-09-08 16:11:23 -070054 */
Jason Gunthorpe2733ea12020-05-01 15:20:48 -030055static inline struct page *hmm_pfn_to_page(unsigned long hmm_pfn)
56{
57 return pfn_to_page(hmm_pfn & ~HMM_PFN_FLAGS);
58}
Jérôme Glissef88a1e92018-04-10 16:29:06 -070059
60/*
Ralph Campbell3b50a6e2020-07-01 15:53:49 -070061 * hmm_pfn_to_map_order() - return the CPU mapping size order
62 *
63 * This is optionally useful to optimize processing of the pfn result
64 * array. It indicates that the page starts at the order aligned VA and is
65 * 1<<order bytes long. Every pfn within an high order page will have the
66 * same pfn flags, both access protections and the map_order. The caller must
67 * be careful with edge cases as the start and end VA of the given page may
68 * extend past the range used with hmm_range_fault().
69 *
70 * This must be called under the caller 'user_lock' after a successful
71 * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
72 * already.
73 */
74static inline unsigned int hmm_pfn_to_map_order(unsigned long hmm_pfn)
75{
76 return (hmm_pfn >> HMM_PFN_ORDER_SHIFT) & 0x1F;
77}
78
79/*
Jérôme Glissef88a1e92018-04-10 16:29:06 -070080 * struct hmm_range - track invalidation lock on virtual address range
81 *
Jason Gunthorpea22dd502019-11-12 16:22:30 -040082 * @notifier: a mmu_interval_notifier that includes the start/end
83 * @notifier_seq: result of mmu_interval_read_begin()
Jérôme Glissef88a1e92018-04-10 16:29:06 -070084 * @start: range virtual start address (inclusive)
85 * @end: range virtual end address (exclusive)
Jason Gunthorpe2733ea12020-05-01 15:20:48 -030086 * @hmm_pfns: array of pfns (big enough for the range)
Jérôme Glisse023a0192019-05-13 17:20:05 -070087 * @default_flags: default flags for the range (write, read, ... see hmm doc)
88 * @pfn_flags_mask: allows to mask pfn flags so that only default_flags matter
Christoph Hellwig08ddddd2020-03-16 20:32:16 +010089 * @dev_private_owner: owner of device private pages
Jérôme Glissef88a1e92018-04-10 16:29:06 -070090 */
91struct hmm_range {
Jason Gunthorpe04ec32f2019-11-12 16:22:20 -040092 struct mmu_interval_notifier *notifier;
93 unsigned long notifier_seq;
Jérôme Glissef88a1e92018-04-10 16:29:06 -070094 unsigned long start;
95 unsigned long end;
Jason Gunthorpe2733ea12020-05-01 15:20:48 -030096 unsigned long *hmm_pfns;
97 unsigned long default_flags;
98 unsigned long pfn_flags_mask;
Christoph Hellwig08ddddd2020-03-16 20:32:16 +010099 void *dev_private_owner;
Jérôme Glissef88a1e92018-04-10 16:29:06 -0700100};
Jérôme Glisse133ff0e2017-09-08 16:11:23 -0700101
102/*
Mike Rapoportee657282022-06-27 09:00:26 +0300103 * Please see Documentation/mm/hmm.rst for how to use the range API.
Jérôme Glisseda4c3c72017-09-08 16:11:31 -0700104 */
Jason Gunthorpebe957c82020-05-01 15:20:45 -0300105int hmm_range_fault(struct hmm_range *range);
Jérôme Glisse74eee182017-09-08 16:11:35 -0700106
107/*
Jérôme Glissea3e0d412019-05-13 17:20:01 -0700108 * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range
Jérôme Glisse74eee182017-09-08 16:11:35 -0700109 *
Jérôme Glissea3e0d412019-05-13 17:20:01 -0700110 * When waiting for mmu notifiers we need some kind of time out otherwise we
Zhen Lei06c88392021-07-07 18:08:19 -0700111 * could potentially wait for ever, 1000ms ie 1s sounds like a long time to
Jérôme Glissea3e0d412019-05-13 17:20:01 -0700112 * wait already.
Jérôme Glisse74eee182017-09-08 16:11:35 -0700113 */
Jérôme Glissea3e0d412019-05-13 17:20:01 -0700114#define HMM_RANGE_DEFAULT_TIMEOUT 1000
115
Jérôme Glisse133ff0e2017-09-08 16:11:23 -0700116#endif /* LINUX_HMM_H */