Miroirs/SlackBuildsOrg
1
0
Fork 0
mirror of https://github.com/SlackBuildsOrg/slackbuilds synced 2024-09-30 05:39:56 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions
SlackBuildsOrg/system/xen/xsa/xsa273-d757c29ffe2e31b15397e43cd58da88b6318b654.patch
Mario Preksavec 9be84725e7
system/xen: Updated for version 4.11.0 
Signed-off-by: Mario Preksavec <mario@slackware.hr>
2018-09-01 07:32:30 +07:00

4115 lines
142 KiB
Diff
Raw Blame History

diff --git a/docs/man/xl.conf.pod.5 b/docs/man/xl.conf.pod.5
index da91b8626c..37262a7ef8 100644
--- a/docs/man/xl.conf.pod.5
+++ b/docs/man/xl.conf.pod.5
@@ -185,6 +185,28 @@ massively huge guests).
=back
+=item B<vm.cpumask>="CPULIST"
+
+=item B<vm.hvm.cpumask>="CPULIST"
+
+=item B<vm.pv.cpumask>="CPULIST"
+
+Global masks that are applied when creating guests and pinning vcpus
+to indicate which cpus they are allowed to run on. Specifically,
+C<vm.cpumask> applies to all guest types, C<vm.hvm.cpumask> applies to
+both HVM and PVH guests and C<vm.pv.cpumask> applies to PV guests.
+
+The hard affinity of guest's vcpus are logical-AND'ed with respective
+masks. If the resulting affinity mask is empty, operation will fail.
+
+Use --ignore-global-affinity-masks to skip applying global masks.
+
+The default value for these masks are all 1's, i.e. all cpus are allowed.
+
+Due to bug(s), these options may not interact well with other options
+concerning CPU affinity. One example is CPU pools. Users should always double
+check that the required affinity has taken effect.
+
=back
=head1 SEE ALSO
diff --git a/docs/misc/xen-command-line.markdown b/docs/misc/xen-command-line.markdown
index 075e5ea159..0886706368 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -489,10 +489,10 @@ accounting for hardware capabilities as enumerated via CPUID.
Currently accepted:
-The Speculation Control hardware features `ibrsb`, `stibp`, `ibpb`, `ssbd` are
-used by default if available and applicable. They can be ignored,
-e.g. `no-ibrsb`, at which point Xen won't use them itself, and won't offer
-them to guests.
+The Speculation Control hardware features `ibrsb`, `stibp`, `ibpb`,
+`l1d-flush` and `ssbd` are used by default if available and applicable. They can
+be ignored, e.g. `no-ibrsb`, at which point Xen won't use them itself, and
+won't offer them to guests.
### cpuid\_mask\_cpu (AMD only)
> `= fam_0f_rev_c | fam_0f_rev_d | fam_0f_rev_e | fam_0f_rev_f | fam_0f_rev_g | fam_10_rev_b | fam_10_rev_c | fam_11_rev_b`
@@ -936,6 +936,8 @@ version are 1 and 2.
use of grant table v2 without transitive grants is an ABI breakage from the
guests point of view.
+The usage of gnttab v2 is not security supported on ARM platforms.
+
### gnttab\_max\_frames
> `= <integer>`
@@ -1544,6 +1546,30 @@ do; there may be other custom operating systems which do. If you're
certain you don't plan on having PV guests which use this feature,
turning it off can reduce the attack surface.
+### pv-l1tf (x86)
+> `= List of [ <bool>, dom0=<bool>, domu=<bool> ]`
+
+> Default: `false` on believed-unaffected hardware, or in pv-shim mode.
+> `domu` on believed-affected hardware.
+
+Mitigations for L1TF / XSA-273 / CVE-2018-3620 for PV guests.
+
+For backwards compatibility, we may not alter an architecturally-legitimate
+pagetable entry a PV guest chooses to write. We can however force such a
+guest into shadow mode so that Xen controls the PTEs which are reachable by
+the CPU pagewalk.
+
+Shadowing is performed at the point where a PV guest first tries to write an
+L1TF-vulnerable PTE. Therefore, a PV guest kernel which has been updated with
+its own L1TF mitigations will not trigger shadow mode if it is well behaved.
+
+If CONFIG\_SHADOW\_PAGING is not compiled in, this mitigation instead crashes
+the guest when an L1TF-vulnerable PTE is written, which still allows updated,
+well-behaved PV guests to run, despite Shadow being compiled out.
+
+In the pv-shim case, Shadow is expected to be compiled out, and a malicious
+guest kernel can only leak data from the shim Xen, rather than the host Xen.
+
### pv-shim (x86)
> `= <boolean>`
@@ -1748,6 +1774,13 @@ Use `smap=hvm` to allow SMAP use by HVM guests only.
Flag to enable Supervisor Mode Execution Protection
Use `smep=hvm` to allow SMEP use by HVM guests only.
+### smt (x86)
+> `= <boolean>`
+
+Default: `true`
+
+Control bring up of multiple hyper-threads per CPU core.
+
### snb\_igd\_quirk
> `= <boolean> | cap | <integer>`
@@ -1758,7 +1791,8 @@ false disable the quirk workaround, which is also the default.
### spec-ctrl (x86)
> `= List of [ <bool>, xen=<bool>, {pv,hvm,msr-sc,rsb}=<bool>,
-> bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,eager-fpu}=<bool> ]`
+> bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,eager-fpu,
+> l1d-flush}=<bool> ]`
Controls for speculative execution sidechannel mitigations. By default, Xen
will pick the most appropriate mitigations based on compiled in support,
@@ -1770,10 +1804,15 @@ extreme care.**
An overall boolean value, `spec-ctrl=no`, can be specified to turn off all
mitigations, including pieces of infrastructure used to virtualise certain
-mitigation features for guests. Alternatively, a slightly more restricted
-`spec-ctrl=no-xen` can be used to turn off all of Xen's mitigations, while
-leaving the virtualisation support in place for guests to use. Use of a
-positive boolean value for either of these options is invalid.
+mitigation features for guests. This also includes settings which `xpti`,
+`smt`, `pv-l1tf` control, unless the respective option(s) have been
+specified earlier on the command line.
+
+Alternatively, a slightly more restricted `spec-ctrl=no-xen` can be used to
+turn off all of Xen's mitigations, while leaving the virtualisation support
+in place for guests to use.
+
+Use of a positive boolean value for either of these options is invalid.
The booleans `pv=`, `hvm=`, `msr-sc=` and `rsb=` offer fine grained control
over the alternative blocks used by Xen. These impact Xen's ability to
@@ -1813,6 +1852,12 @@ from using fully eager FPU context switches. This is currently implemented as
a global control. By default, Xen will choose to use fully eager context
switches on hardware believed to speculate past #NM exceptions.
+On hardware supporting L1D_FLUSH, the `l1d-flush=` option can be used to force
+or prevent Xen from issuing an L1 data cache flush on each VMEntry.
+Irrespective of Xen's setting, the feature is virtualised for HVM guests to
+use. By default, Xen will enable this mitigation on hardware believed to be
+vulnerable to L1TF.
+
### sync\_console
> `= <boolean>`
diff --git a/tools/examples/xl.conf b/tools/examples/xl.conf
index 374b6bbc2e..0446deb304 100644
--- a/tools/examples/xl.conf
+++ b/tools/examples/xl.conf
@@ -37,3 +37,8 @@
# (which can take a long time to find out if launching huge guests).
# see xl.conf(5) for details.
#claim_mode=1
+
+# Specify global vcpu hard affinity masks. See xl.conf(5) for details.
+#vm.cpumask="0-7"
+#vm.pv.cpumask="0-3"
+#vm.hvm.cpumask="3-7"
diff --git a/tools/libxl/libxl_cpuid.c b/tools/libxl/libxl_cpuid.c
index 7b0f594c3d..52e16c20ed 100644
--- a/tools/libxl/libxl_cpuid.c
+++ b/tools/libxl/libxl_cpuid.c
@@ -204,6 +204,7 @@ int libxl_cpuid_parse_config(libxl_cpuid_policy_list *cpuid, const char* str)
{"avx512-4fmaps",0x00000007, 0, CPUID_REG_EDX, 3, 1},
{"ibrsb", 0x00000007, 0, CPUID_REG_EDX, 26, 1},
{"stibp", 0x00000007, 0, CPUID_REG_EDX, 27, 1},
+ {"l1d-flush", 0x00000007, 0, CPUID_REG_EDX, 28, 1},
{"arch-caps", 0x00000007, 0, CPUID_REG_EDX, 29, 1},
{"ssbd", 0x00000007, 0, CPUID_REG_EDX, 31, 1},
diff --git a/tools/misc/xen-cpuid.c b/tools/misc/xen-cpuid.c
index e116339733..3888b4e158 100644
--- a/tools/misc/xen-cpuid.c
+++ b/tools/misc/xen-cpuid.c
@@ -143,7 +143,7 @@ static const char *str_7d0[32] =
[ 2] = "avx512_4vnniw", [ 3] = "avx512_4fmaps",
[26] = "ibrsb", [27] = "stibp",
- /* 28 */ [29] = "arch_caps",
+ [28] = "l1d_flush", [29] = "arch_caps",
/* 30 */ [31] = "ssbd",
};
diff --git a/tools/ocaml/xenstored/store.ml b/tools/ocaml/xenstored/store.ml
index 13cf3b5bf4..5a8c377603 100644
--- a/tools/ocaml/xenstored/store.ml
+++ b/tools/ocaml/xenstored/store.ml
@@ -262,7 +262,8 @@ let path_write store perm path value =
Node.check_perm store.root perm Perms.WRITE;
Node.set_value store.root value, false
) else
- Path.apply_modify store.root path do_write, !node_created
+ let root = Path.apply_modify store.root path do_write in
+ root, !node_created
let path_rm store perm path =
let do_rm node name =
diff --git a/tools/xl/xl.c b/tools/xl/xl.c
index 179908b4f6..7d2142f16f 100644
--- a/tools/xl/xl.c
+++ b/tools/xl/xl.c
@@ -28,6 +28,9 @@
#include <libxl_utils.h>
#include <libxlutil.h>
#include "xl.h"
+#include "xl_parse.h"
+
+#include "xl_utils.h"
xentoollog_logger_stdiostream *logger;
int dryrun_only;
@@ -42,6 +45,9 @@ char *default_gatewaydev = NULL;
char *default_vifbackend = NULL;
char *default_remus_netbufscript = NULL;
char *default_colo_proxy_script = NULL;
+libxl_bitmap global_vm_affinity_mask;
+libxl_bitmap global_hvm_affinity_mask;
+libxl_bitmap global_pv_affinity_mask;
enum output_format default_output_format = OUTPUT_FORMAT_JSON;
int claim_mode = 1;
bool progress_use_cr = 0;
@@ -203,6 +209,26 @@ static void parse_global_config(const char *configfile,
if (!xlu_cfg_get_long (config, "max_maptrack_frames", &l, 0))
max_maptrack_frames = l;
+ libxl_bitmap_init(&global_vm_affinity_mask);
+ libxl_cpu_bitmap_alloc(ctx, &global_vm_affinity_mask, 0);
+ libxl_bitmap_init(&global_hvm_affinity_mask);
+ libxl_cpu_bitmap_alloc(ctx, &global_hvm_affinity_mask, 0);
+ libxl_bitmap_init(&global_pv_affinity_mask);
+ libxl_cpu_bitmap_alloc(ctx, &global_pv_affinity_mask, 0);
+
+ if (!xlu_cfg_get_string (config, "vm.cpumask", &buf, 0))
+ parse_cpurange(buf, &global_vm_affinity_mask);
+ else
+ libxl_bitmap_set_any(&global_vm_affinity_mask);
+ if (!xlu_cfg_get_string (config, "vm.hvm.cpumask", &buf, 0))
+ parse_cpurange(buf, &global_hvm_affinity_mask);
+ else
+ libxl_bitmap_set_any(&global_hvm_affinity_mask);
+ if (!xlu_cfg_get_string (config, "vm.pv.cpumask", &buf, 0))
+ parse_cpurange(buf, &global_pv_affinity_mask);
+ else
+ libxl_bitmap_set_any(&global_pv_affinity_mask);
+
xlu_cfg_destroy(config);
}
diff --git a/tools/xl/xl.h b/tools/xl/xl.h
index 4e784ff402..7e97144b50 100644
--- a/tools/xl/xl.h
+++ b/tools/xl/xl.h
@@ -41,6 +41,7 @@ struct domain_create {
int vncautopass;
int console_autoconnect;
int checkpointed_stream;
+ int ignore_global_affinity_masks;
const char *config_file;
char *extra_config; /* extra config string */
const char *restore_file;
@@ -279,6 +280,9 @@ extern char *default_colo_proxy_script;
extern char *blkdev_start;
extern int max_grant_frames;
extern int max_maptrack_frames;
+extern libxl_bitmap global_vm_affinity_mask;
+extern libxl_bitmap global_hvm_affinity_mask;
+extern libxl_bitmap global_pv_affinity_mask;
enum output_format {
OUTPUT_FORMAT_JSON,
@@ -294,6 +298,9 @@ typedef enum {
} domain_restart_type;
extern void printf_info_sexp(int domid, libxl_domain_config *d_config, FILE *fh);
+extern void apply_global_affinity_masks(libxl_domain_type type,
+ libxl_bitmap *vcpu_affinity_array,
+ unsigned int size);
#define XL_GLOBAL_CONFIG XEN_CONFIG_DIR "/xl.conf"
#define XL_LOCK_FILE XEN_LOCK_DIR "/xl"
diff --git a/tools/xl/xl_cmdtable.c b/tools/xl/xl_cmdtable.c
index bf2ced8140..54c2db6022 100644
--- a/tools/xl/xl_cmdtable.c
+++ b/tools/xl/xl_cmdtable.c
@@ -34,7 +34,8 @@ struct cmd_spec cmd_table[] = {
"-e Do not wait in the background for the death of the domain.\n"
"-V, --vncviewer Connect to the VNC display after the domain is created.\n"
"-A, --vncviewer-autopass\n"
- " Pass VNC password to viewer via stdin."
+ " Pass VNC password to viewer via stdin.\n"
+ "--ignore-global-affinity-masks Ignore global masks in xl.conf."
},
{ "config-update",
&main_config_update, 1, 1,
@@ -224,7 +225,8 @@ struct cmd_spec cmd_table[] = {
&main_vcpupin, 1, 1,
"Set which CPUs a VCPU can use",
"[option] <Domain> <VCPU|all> <Hard affinity|-|all> <Soft affinity|-|all>",
- "-f, --force undo an override pinning done by the kernel",
+ "-f, --force undo an override pinning done by the kernel\n"
+ "--ignore-global-affinity-masks Ignore global masks in xl.conf",
},
{ "vcpu-set",
&main_vcpuset, 0, 1,
diff --git a/tools/xl/xl_vcpu.c b/tools/xl/xl_vcpu.c
index 8e735b38c1..3384eeed06 100644
--- a/tools/xl/xl_vcpu.c
+++ b/tools/xl/xl_vcpu.c
@@ -68,6 +68,61 @@ static void print_domain_vcpuinfo(uint32_t domid, uint32_t nr_cpus)
libxl_vcpuinfo_list_free(vcpuinfo, nb_vcpu);
}
+void apply_global_affinity_masks(libxl_domain_type type,
+ libxl_bitmap *vcpu_affinity_array,
+ unsigned int size)
+{
+ libxl_bitmap *mask = &global_vm_affinity_mask;
+ libxl_bitmap *type_mask;
+ unsigned int i;
+
+ switch (type) {
+ case LIBXL_DOMAIN_TYPE_HVM:
+ case LIBXL_DOMAIN_TYPE_PVH:
+ type_mask = &global_hvm_affinity_mask;
+ break;
+ case LIBXL_DOMAIN_TYPE_PV:
+ type_mask = &global_pv_affinity_mask;
+ break;
+ default:
+ fprintf(stderr, "Unknown guest type\n");
+ exit(EXIT_FAILURE);
+ }
+
+ for (i = 0; i < size; i++) {
+ int rc;
+ libxl_bitmap *t = &vcpu_affinity_array[i];
+ libxl_bitmap b1, b2;
+
+ libxl_bitmap_init(&b1);
+ libxl_bitmap_init(&b2);
+
+ rc = libxl_bitmap_and(ctx, &b1, t, mask);
+ if (rc) {
+ fprintf(stderr, "libxl_bitmap_and errored\n");
+ exit(EXIT_FAILURE);
+ }
+ rc = libxl_bitmap_and(ctx, &b2, &b1, type_mask);
+ if (rc) {
+ fprintf(stderr, "libxl_bitmap_and errored\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (libxl_bitmap_is_empty(&b2)) {
+ fprintf(stderr, "vcpu hard affinity map is empty\n");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Replace target bitmap with the result */
+ libxl_bitmap_dispose(t);
+ libxl_bitmap_init(t);
+ libxl_bitmap_copy_alloc(ctx, t, &b2);
+
+ libxl_bitmap_dispose(&b1);
+ libxl_bitmap_dispose(&b2);
+ }
+}
+
static void vcpulist(int argc, char **argv)
{
libxl_dominfo *dominfo;
@@ -118,6 +173,7 @@ int main_vcpupin(int argc, char **argv)
{
static struct option opts[] = {
{"force", 0, 0, 'f'},
+ {"ignore-global-affinity-masks", 0, 0, 'i'},
COMMON_LONG_OPTS
};
libxl_vcpuinfo *vcpuinfo;
@@ -132,15 +188,18 @@ int main_vcpupin(int argc, char **argv)
const char *vcpu, *hard_str, *soft_str;
char *endptr;
int opt, nb_cpu, nb_vcpu, rc = EXIT_FAILURE;
- bool force = false;
+ bool force = false, ignore_masks = false;
libxl_bitmap_init(&cpumap_hard);
libxl_bitmap_init(&cpumap_soft);
- SWITCH_FOREACH_OPT(opt, "f", opts, "vcpu-pin", 3) {
+ SWITCH_FOREACH_OPT(opt, "fi", opts, "vcpu-pin", 3) {
case 'f':
force = true;
break;
+ case 'i':
+ ignore_masks = true;
+ break;
default:
break;
}
@@ -222,6 +281,23 @@ int main_vcpupin(int argc, char **argv)
goto out;
}
+ /* Only hard affinity matters here */
+ if (!ignore_masks) {
+ libxl_domain_config d_config;
+
+ libxl_domain_config_init(&d_config);
+ rc = libxl_retrieve_domain_configuration(ctx, domid, &d_config);
+ if (rc) {
+ fprintf(stderr, "Could not retrieve domain configuration\n");
+ libxl_domain_config_dispose(&d_config);
+ goto out;
+ }
+
+ apply_global_affinity_masks(d_config.b_info.type, hard, 1);
+
+ libxl_domain_config_dispose(&d_config);
+ }
+
if (force) {
if (libxl_set_vcpuaffinity_force(ctx, domid, vcpuid, hard, soft)) {
fprintf(stderr, "Could not set affinity for vcpu `%ld'.\n",
diff --git a/tools/xl/xl_vmcontrol.c b/tools/xl/xl_vmcontrol.c
index 89c2b25ded..a1d633795c 100644
--- a/tools/xl/xl_vmcontrol.c
+++ b/tools/xl/xl_vmcontrol.c
@@ -804,6 +804,36 @@ int create_domain(struct domain_create *dom_info)
parse_config_data(config_source, config_data, config_len, &d_config);
}
+ if (!dom_info->ignore_global_affinity_masks) {
+ libxl_domain_build_info *b_info = &d_config.b_info;
+
+ /* It is possible that no hard affinity is specified in config file.
+ * Generate hard affinity maps now if we care about those.
+ */
+ if (b_info->num_vcpu_hard_affinity == 0 &&
+ (!libxl_bitmap_is_full(&global_vm_affinity_mask) ||
+ (b_info->type == LIBXL_DOMAIN_TYPE_PV &&
+ !libxl_bitmap_is_full(&global_pv_affinity_mask)) ||
+ (b_info->type != LIBXL_DOMAIN_TYPE_PV &&
+ !libxl_bitmap_is_full(&global_hvm_affinity_mask))
+ )) {
+ b_info->num_vcpu_hard_affinity = b_info->max_vcpus;
+ b_info->vcpu_hard_affinity =
+ xmalloc(b_info->max_vcpus * sizeof(libxl_bitmap));
+
+ for (i = 0; i < b_info->num_vcpu_hard_affinity; i++) {
+ libxl_bitmap *m = &b_info->vcpu_hard_affinity[i];
+ libxl_bitmap_init(m);
+ libxl_cpu_bitmap_alloc(ctx, m, 0);
+ libxl_bitmap_set_any(m);
+ }
+ }
+
+ apply_global_affinity_masks(b_info->type,
+ b_info->vcpu_hard_affinity,
+ b_info->num_vcpu_hard_affinity);
+ }
+
if (migrate_fd >= 0) {
if (d_config.c_info.name) {
/* when we receive a domain we get its name from the config
@@ -1124,7 +1154,7 @@ int main_create(int argc, char **argv)
const char *filename = NULL;
struct domain_create dom_info;
int paused = 0, debug = 0, daemonize = 1, console_autoconnect = 0,
- quiet = 0, monitor = 1, vnc = 0, vncautopass = 0;
+ quiet = 0, monitor = 1, vnc = 0, vncautopass = 0, ignore_masks = 0;
int opt, rc;
static struct option opts[] = {
{"dryrun", 0, 0, 'n'},
@@ -1132,6 +1162,7 @@ int main_create(int argc, char **argv)
{"defconfig", 1, 0, 'f'},
{"vncviewer", 0, 0, 'V'},
{"vncviewer-autopass", 0, 0, 'A'},
+ {"ignore-global-affinity-masks", 0, 0, 'i'},
COMMON_LONG_OPTS
};
@@ -1142,7 +1173,7 @@ int main_create(int argc, char **argv)
argc--; argv++;
}
- SWITCH_FOREACH_OPT(opt, "Fnqf:pcdeVA", opts, "create", 0) {
+ SWITCH_FOREACH_OPT(opt, "Fnqf:pcdeVAi", opts, "create", 0) {
case 'f':
filename = optarg;
break;
@@ -1174,6 +1205,9 @@ int main_create(int argc, char **argv)
case 'A':
vnc = vncautopass = 1;
break;
+ case 'i':
+ ignore_masks = 1;
+ break;
}
memset(&dom_info, 0, sizeof(dom_info));
@@ -1203,6 +1237,7 @@ int main_create(int argc, char **argv)
dom_info.vnc = vnc;
dom_info.vncautopass = vncautopass;
dom_info.console_autoconnect = console_autoconnect;
+ dom_info.ignore_global_affinity_masks = ignore_masks;
rc = create_domain(&dom_info);
if (rc < 0) {
#diff --git a/xen/Makefile b/xen/Makefile
#index 4d075c381f..a922a1b7b5 100644
#--- a/xen/Makefile
#+++ b/xen/Makefile
#@@ -2,7 +2,7 @@
# # All other places this is stored (eg. compile.h) should be autogenerated.
# export XEN_VERSION = 4
# export XEN_SUBVERSION = 11
#-export XEN_EXTRAVERSION ?= .0$(XEN_VENDORVERSION)
#+export XEN_EXTRAVERSION ?= .1-pre$(XEN_VENDORVERSION)
# export XEN_FULLVERSION = $(XEN_VERSION).$(XEN_SUBVERSION)$(XEN_EXTRAVERSION)
# -include xen-version
#
diff --git a/xen/arch/x86/Kconfig b/xen/arch/x86/Kconfig
index f64fc56739..cfba4a708c 100644
--- a/xen/arch/x86/Kconfig
+++ b/xen/arch/x86/Kconfig
@@ -72,6 +72,7 @@ config SHADOW_PAGING
* Running HVM guests on hardware lacking hardware paging support
(First-generation Intel VT-x or AMD SVM).
* Live migration of PV guests.
+ * L1TF sidechannel mitigation for PV guests.
Under a small number of specific workloads, shadow paging may be
deliberately used as a performance optimisation.
diff --git a/xen/arch/x86/cpu/amd.c b/xen/arch/x86/cpu/amd.c
index 458a3fe60c..76078b55b2 100644
--- a/xen/arch/x86/cpu/amd.c
+++ b/xen/arch/x86/cpu/amd.c
@@ -505,17 +505,23 @@ static void amd_get_topology(struct cpuinfo_x86 *c)
u32 eax, ebx, ecx, edx;
cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
- c->compute_unit_id = ebx & 0xFF;
c->x86_num_siblings = ((ebx >> 8) & 0x3) + 1;
+
+ if (c->x86 < 0x17)
+ c->compute_unit_id = ebx & 0xFF;
+ else {
+ c->cpu_core_id = ebx & 0xFF;
+ c->x86_max_cores /= c->x86_num_siblings;
+ }
}
if (opt_cpu_info)
printk("CPU %d(%d) -> Processor %d, %s %d\n",
cpu, c->x86_max_cores, c->phys_proc_id,
- cpu_has(c, X86_FEATURE_TOPOEXT) ? "Compute Unit" :
- "Core",
- cpu_has(c, X86_FEATURE_TOPOEXT) ? c->compute_unit_id :
- c->cpu_core_id);
+ c->compute_unit_id != INVALID_CUID ? "Compute Unit"
+ : "Core",
+ c->compute_unit_id != INVALID_CUID ? c->compute_unit_id
+ : c->cpu_core_id);
}
static void early_init_amd(struct cpuinfo_x86 *c)
diff --git a/xen/arch/x86/cpu/common.c b/xen/arch/x86/cpu/common.c
index 528aff1811..fdb022875a 100644
--- a/xen/arch/x86/cpu/common.c
+++ b/xen/arch/x86/cpu/common.c
@@ -14,6 +14,7 @@
#include <public/sysctl.h> /* for XEN_INVALID_{SOCKET,CORE}_ID */
#include "cpu.h"
+#include "mcheck/x86_mca.h"
bool_t opt_arat = 1;
boolean_param("arat", opt_arat);
@@ -355,6 +356,9 @@ static void __init early_cpu_detect(void)
hap_paddr_bits = PADDR_BITS;
}
+ if (c->x86_vendor != X86_VENDOR_AMD)
+ park_offline_cpus = opt_mce;
+
initialize_cpu_data(0);
}
diff --git a/xen/arch/x86/cpu/mcheck/mce.c b/xen/arch/x86/cpu/mcheck/mce.c
index a8c287d124..32273d9208 100644
--- a/xen/arch/x86/cpu/mcheck/mce.c
+++ b/xen/arch/x86/cpu/mcheck/mce.c
@@ -692,12 +692,15 @@ static void cpu_bank_free(unsigned int cpu)
mcabanks_free(poll);
mcabanks_free(clr);
+
+ per_cpu(poll_bankmask, cpu) = NULL;
+ per_cpu(mce_clear_banks, cpu) = NULL;
}
static int cpu_bank_alloc(unsigned int cpu)
{
- struct mca_banks *poll = mcabanks_alloc();
- struct mca_banks *clr = mcabanks_alloc();
+ struct mca_banks *poll = per_cpu(poll_bankmask, cpu) ?: mcabanks_alloc();
+ struct mca_banks *clr = per_cpu(mce_clear_banks, cpu) ?: mcabanks_alloc();
if ( !poll || !clr )
{
@@ -725,7 +728,13 @@ static int cpu_callback(
case CPU_UP_CANCELED:
case CPU_DEAD:
- cpu_bank_free(cpu);
+ if ( !park_offline_cpus )
+ cpu_bank_free(cpu);
+ break;
+
+ case CPU_REMOVE:
+ if ( park_offline_cpus )
+ cpu_bank_free(cpu);
break;
}
diff --git a/xen/arch/x86/cpu/mcheck/mce_intel.c b/xen/arch/x86/cpu/mcheck/mce_intel.c
index e5dd956a24..4474a34e34 100644
--- a/xen/arch/x86/cpu/mcheck/mce_intel.c
+++ b/xen/arch/x86/cpu/mcheck/mce_intel.c
@@ -636,8 +636,6 @@ static void clear_cmci(void)
static void cpu_mcheck_disable(void)
{
- clear_in_cr4(X86_CR4_MCE);
-
if ( cmci_support && opt_mce )
clear_cmci();
}
diff --git a/xen/arch/x86/cpu/vpmu_intel.c b/xen/arch/x86/cpu/vpmu_intel.c
index 207e2e712c..6e27f6ec8e 100644
--- a/xen/arch/x86/cpu/vpmu_intel.c
+++ b/xen/arch/x86/cpu/vpmu_intel.c
@@ -454,13 +454,11 @@ static int core2_vpmu_alloc_resource(struct vcpu *v)
if ( is_hvm_vcpu(v) )
{
- wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
- if ( vmx_add_host_load_msr(MSR_CORE_PERF_GLOBAL_CTRL) )
+ if ( vmx_add_host_load_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, 0) )
goto out_err;
- if ( vmx_add_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL) )
+ if ( vmx_add_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, 0) )
goto out_err;
- vmx_write_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
}
core2_vpmu_cxt = xzalloc_bytes(sizeof(*core2_vpmu_cxt) +
@@ -535,27 +533,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content,
uint64_t *enabled_cntrs;
if ( !core2_vpmu_msr_common_check(msr, &type, &index) )
- {
- /* Special handling for BTS */
- if ( msr == MSR_IA32_DEBUGCTLMSR )
- {
- supported |= IA32_DEBUGCTLMSR_TR | IA32_DEBUGCTLMSR_BTS |
- IA32_DEBUGCTLMSR_BTINT;
-
- if ( cpu_has(&current_cpu_data, X86_FEATURE_DSCPL) )
- supported |= IA32_DEBUGCTLMSR_BTS_OFF_OS |
- IA32_DEBUGCTLMSR_BTS_OFF_USR;
- if ( !(msr_content & ~supported) &&
- vpmu_is_set(vpmu, VPMU_CPU_HAS_BTS) )
- return 0;
- if ( (msr_content & supported) &&
- !vpmu_is_set(vpmu, VPMU_CPU_HAS_BTS) )
- printk(XENLOG_G_WARNING
- "%pv: Debug Store unsupported on this CPU\n",
- current);
- }
return -EINVAL;
- }
ASSERT(!supported);
@@ -613,7 +591,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content,
return -EINVAL;
if ( is_hvm_vcpu(v) )
- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL,
+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL,
&core2_vpmu_cxt->global_ctrl);
else
rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, core2_vpmu_cxt->global_ctrl);
@@ -682,7 +660,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content,
return -EINVAL;
if ( is_hvm_vcpu(v) )
- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL,
+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL,
&core2_vpmu_cxt->global_ctrl);
else
rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, core2_vpmu_cxt->global_ctrl);
@@ -701,7 +679,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content,
else
{
if ( is_hvm_vcpu(v) )
- vmx_write_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, msr_content);
+ vmx_write_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, msr_content);
else
wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, msr_content);
}
@@ -735,7 +713,7 @@ static int core2_vpmu_do_rdmsr(unsigned int msr, uint64_t *msr_content)
break;
case MSR_CORE_PERF_GLOBAL_CTRL:
if ( is_hvm_vcpu(v) )
- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, msr_content);
+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, msr_content);
else
rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, *msr_content);
break;
diff --git a/xen/arch/x86/cpuid.c b/xen/arch/x86/cpuid.c
index beee47d0ed..5cc89e2b34 100644
--- a/xen/arch/x86/cpuid.c
+++ b/xen/arch/x86/cpuid.c
@@ -43,6 +43,11 @@ static int __init parse_xen_cpuid(const char *s)
if ( !val )
setup_clear_cpu_cap(X86_FEATURE_STIBP);
}
+ else if ( (val = parse_boolean("l1d-flush", s, ss)) >= 0 )
+ {
+ if ( !val )
+ setup_clear_cpu_cap(X86_FEATURE_L1D_FLUSH);
+ }
else if ( (val = parse_boolean("ssbd", s, ss)) >= 0 )
{
if ( !val )
diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 9850a782ec..c39cf2c6e5 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -107,10 +107,11 @@ static void play_dead(void)
local_irq_disable();
/*
- * NOTE: After cpu_exit_clear, per-cpu variables are no longer accessible,
- * as they may be freed at any time. In this case, heap corruption or
- * #PF can occur (when heap debugging is enabled). For example, even
- * printk() can involve tasklet scheduling, which touches per-cpu vars.
+ * NOTE: After cpu_exit_clear, per-cpu variables may no longer accessible,
+ * as they may be freed at any time if offline CPUs don't get parked. In
+ * this case, heap corruption or #PF can occur (when heap debugging is
+ * enabled). For example, even printk() can involve tasklet scheduling,
+ * which touches per-cpu vars.
*
* Consider very carefully when adding code to *dead_idle. Most hypervisor
* subsystems are unsafe to call.
diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
index 8fbbf3aeb3..dd91038a67 100644
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -225,7 +225,8 @@ static int update_domain_cpuid_info(struct domain *d,
*/
call_policy_changed = (is_hvm_domain(d) &&
((old_7d0 ^ p->feat.raw[0].d) &
- cpufeat_mask(X86_FEATURE_IBRSB)));
+ (cpufeat_mask(X86_FEATURE_IBRSB) |
+ cpufeat_mask(X86_FEATURE_L1D_FLUSH))));
break;
case 0xa:
@@ -1163,7 +1164,7 @@ long arch_do_domctl(
if ( _xcr0_accum )
{
if ( evc->size >= PV_XSAVE_HDR_SIZE + XSTATE_AREA_MIN_SIZE )
- ret = validate_xstate(_xcr0, _xcr0_accum,
+ ret = validate_xstate(d, _xcr0, _xcr0_accum,
&_xsave_area->xsave_hdr);
}
else if ( !_xcr0 )
@@ -1187,8 +1188,7 @@ long arch_do_domctl(
vcpu_pause(v);
v->arch.xcr0 = _xcr0;
v->arch.xcr0_accum = _xcr0_accum;
- if ( _xcr0_accum & XSTATE_NONLAZY )
- v->arch.nonlazy_xstate_used = 1;
+ v->arch.nonlazy_xstate_used = _xcr0_accum & XSTATE_NONLAZY;
compress_xsave_states(v, _xsave_area,
evc->size - PV_XSAVE_HDR_SIZE);
vcpu_unpause(v);
diff --git a/xen/arch/x86/genapic/x2apic.c b/xen/arch/x86/genapic/x2apic.c
index 4779b0d0d5..d997806272 100644
--- a/xen/arch/x86/genapic/x2apic.c
+++ b/xen/arch/x86/genapic/x2apic.c
@@ -201,18 +201,21 @@ static int update_clusterinfo(
if ( !cluster_cpus_spare )
cluster_cpus_spare = xzalloc(cpumask_t);
if ( !cluster_cpus_spare ||
- !alloc_cpumask_var(&per_cpu(scratch_mask, cpu)) )
+ !cond_alloc_cpumask_var(&per_cpu(scratch_mask, cpu)) )
err = -ENOMEM;
break;
case CPU_UP_CANCELED:
case CPU_DEAD:
+ case CPU_REMOVE:
+ if ( park_offline_cpus == (action != CPU_REMOVE) )
+ break;
if ( per_cpu(cluster_cpus, cpu) )
{
cpumask_clear_cpu(cpu, per_cpu(cluster_cpus, cpu));
if ( cpumask_empty(per_cpu(cluster_cpus, cpu)) )
- xfree(per_cpu(cluster_cpus, cpu));
+ XFREE(per_cpu(cluster_cpus, cpu));
}
- free_cpumask_var(per_cpu(scratch_mask, cpu));
+ FREE_CPUMASK_VAR(per_cpu(scratch_mask, cpu));
break;
}
diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c
index c23983cdff..4cbb688c05 100644
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -907,6 +907,9 @@ const char *hvm_efer_valid(const struct vcpu *v, uint64_t value,
else
p = &host_cpuid_policy;
+ if ( value & ~EFER_KNOWN_MASK )
+ return "Unknown bits set";
+
if ( (value & EFER_SCE) && !p->extd.syscall )
return "SCE without feature";
@@ -1269,7 +1272,7 @@ static int hvm_load_cpu_xsave_states(struct domain *d, hvm_domain_context_t *h)
ctxt = (struct hvm_hw_cpu_xsave *)&h->data[h->cur];
h->cur += desc->length;
- err = validate_xstate(ctxt->xcr0, ctxt->xcr0_accum,
+ err = validate_xstate(d, ctxt->xcr0, ctxt->xcr0_accum,
(const void *)&ctxt->save_area.xsave_hdr);
if ( err )
{
@@ -1324,8 +1327,7 @@ static int hvm_load_cpu_xsave_states(struct domain *d, hvm_domain_context_t *h)
v->arch.xcr0 = ctxt->xcr0;
v->arch.xcr0_accum = ctxt->xcr0_accum;
- if ( ctxt->xcr0_accum & XSTATE_NONLAZY )
- v->arch.nonlazy_xstate_used = 1;
+ v->arch.nonlazy_xstate_used = ctxt->xcr0_accum & XSTATE_NONLAZY;
compress_xsave_states(v, &ctxt->save_area,
size - offsetof(struct hvm_hw_cpu_xsave, save_area));
diff --git a/xen/arch/x86/hvm/svm/svm.c b/xen/arch/x86/hvm/svm/svm.c
index 165500e3f2..b964c59dad 100644
--- a/xen/arch/x86/hvm/svm/svm.c
+++ b/xen/arch/x86/hvm/svm/svm.c
@@ -1432,24 +1432,18 @@ static void svm_inject_event(const struct x86_event *event)
* Xen must emulate enough of the event injection to be sure that a
* further fault shouldn't occur during delivery. This covers the fact
* that hardware doesn't perform DPL checking on injection.
- *
- * Also, it accounts for proper positioning of %rip for an event with trap
- * semantics (where %rip should point after the instruction) which suffers
- * a fault during injection (at which point %rip should point at the
- * instruction).
*/
if ( event->type == X86_EVENTTYPE_PRI_SW_EXCEPTION ||
- (!cpu_has_svm_nrips && (event->type == X86_EVENTTYPE_SW_INTERRUPT ||
- event->type == X86_EVENTTYPE_SW_EXCEPTION)) )
+ (!cpu_has_svm_nrips && (event->type >= X86_EVENTTYPE_SW_INTERRUPT)) )
svm_emul_swint_injection(&_event);
- switch ( _event.vector )
+ switch ( _event.vector | -(_event.type == X86_EVENTTYPE_SW_INTERRUPT) )
{
case TRAP_debug:
if ( regs->eflags & X86_EFLAGS_TF )
{
__restore_debug_registers(vmcb, curr);
- vmcb_set_dr6(vmcb, vmcb_get_dr6(vmcb) | 0x4000);
+ vmcb_set_dr6(vmcb, vmcb_get_dr6(vmcb) | DR_STEP);
}
/* fall through */
case TRAP_int3:
@@ -1459,6 +1453,13 @@ static void svm_inject_event(const struct x86_event *event)
domain_pause_for_debugger();
return;
}
+ break;
+
+ case TRAP_page_fault:
+ ASSERT(_event.type == X86_EVENTTYPE_HW_EXCEPTION);
+ curr->arch.hvm_vcpu.guest_cr[2] = _event.cr2;
+ vmcb_set_cr2(vmcb, _event.cr2);
+ break;
}
if ( unlikely(eventinj.fields.v) &&
@@ -1481,13 +1482,9 @@ static void svm_inject_event(const struct x86_event *event)
* icebp, software events with trap semantics need emulating, so %rip in
* the trap frame points after the instruction.
*
- * The x86 emulator (if requested by the x86_swint_emulate_* choice) will
- * have performed checks such as presence/dpl/etc and believes that the
- * event injection will succeed without faulting.
- *
- * The x86 emulator will always provide fault semantics for software
- * events, with _trap.insn_len set appropriately. If the injection
- * requires emulation, move %rip forwards at this point.
+ * svm_emul_swint_injection() has already confirmed that events with trap
+ * semantics won't fault on injection. Position %rip/NextRIP suitably,
+ * and restrict the event type to what hardware will tolerate.
*/
switch ( _event.type )
{
@@ -1544,16 +1541,12 @@ static void svm_inject_event(const struct x86_event *event)
eventinj.fields.errorcode == (uint16_t)eventinj.fields.errorcode);
vmcb->eventinj = eventinj;
- if ( _event.vector == TRAP_page_fault )
- {
- curr->arch.hvm_vcpu.guest_cr[2] = _event.cr2;
- vmcb_set_cr2(vmcb, _event.cr2);
- HVMTRACE_LONG_2D(PF_INJECT, _event.error_code, TRC_PAR_LONG(_event.cr2));
- }
+ if ( _event.vector == TRAP_page_fault &&
+ _event.type == X86_EVENTTYPE_HW_EXCEPTION )
+ HVMTRACE_LONG_2D(PF_INJECT, _event.error_code,
+ TRC_PAR_LONG(_event.cr2));
else
- {
HVMTRACE_2D(INJ_EXC, _event.vector, _event.error_code);
- }
}
static int svm_event_pending(struct vcpu *v)
diff --git a/xen/arch/x86/hvm/vmx/entry.S b/xen/arch/x86/hvm/vmx/entry.S
index aa2f103895..afd552f2b9 100644
--- a/xen/arch/x86/hvm/vmx/entry.S
+++ b/xen/arch/x86/hvm/vmx/entry.S
@@ -41,6 +41,15 @@ ENTRY(vmx_asm_vmexit_handler)
SPEC_CTRL_ENTRY_FROM_HVM /* Req: b=curr %rsp=regs/cpuinfo, Clob: acd */
/* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+ /* Hardware clears MSR_DEBUGCTL on VMExit. Reinstate it if debugging Xen. */
+ .macro restore_lbr
+ mov $IA32_DEBUGCTLMSR_LBR, %eax
+ mov $MSR_IA32_DEBUGCTLMSR, %ecx
+ xor %edx, %edx
+ wrmsr
+ .endm
+ ALTERNATIVE "", restore_lbr, X86_FEATURE_XEN_LBR
+
mov %rsp,%rdi
call vmx_vmexit_handler
diff --git a/xen/arch/x86/hvm/vmx/vmcs.c b/xen/arch/x86/hvm/vmx/vmcs.c
index 258fc08f72..2ba0c40808 100644
--- a/xen/arch/x86/hvm/vmx/vmcs.c
+++ b/xen/arch/x86/hvm/vmx/vmcs.c
@@ -38,6 +38,7 @@
#include <asm/flushtlb.h>
#include <asm/monitor.h>
#include <asm/shadow.h>
+#include <asm/spec_ctrl.h>
#include <asm/tboot.h>
#include <asm/apic.h>
@@ -996,6 +997,7 @@ static int construct_vmcs(struct vcpu *v)
struct domain *d = v->domain;
u32 vmexit_ctl = vmx_vmexit_control;
u32 vmentry_ctl = vmx_vmentry_control;
+ int rc = 0;
vmx_vmcs_enter(v);
@@ -1083,8 +1085,8 @@ static int construct_vmcs(struct vcpu *v)
if ( msr_bitmap == NULL )
{
- vmx_vmcs_exit(v);
- return -ENOMEM;
+ rc = -ENOMEM;
+ goto out;
}
memset(msr_bitmap, ~0, PAGE_SIZE);
@@ -1268,141 +1270,197 @@ static int construct_vmcs(struct vcpu *v)
if ( cpu_has_vmx_tsc_scaling )
__vmwrite(TSC_MULTIPLIER, d->arch.hvm_domain.tsc_scaling_ratio);
- vmx_vmcs_exit(v);
-
/* will update HOST & GUEST_CR3 as reqd */
paging_update_paging_modes(v);
vmx_vlapic_msr_changed(v);
- return 0;
+ if ( opt_l1d_flush && paging_mode_hap(d) )
+ rc = vmx_add_msr(v, MSR_FLUSH_CMD, FLUSH_CMD_L1D,
+ VMX_MSR_GUEST_LOADONLY);
+
+ out:
+ vmx_vmcs_exit(v);
+
+ return rc;
}
-static int vmx_msr_entry_key_cmp(const void *key, const void *elt)
+/*
+ * Search an MSR list looking for an MSR entry, or the slot in which it should
+ * live (to keep the data sorted) if an entry is not found.
+ *
+ * The return pointer is guaranteed to be bounded by start and end. However,
+ * it may point at end, and may be invalid for the caller to dereference.
+ */
+static struct vmx_msr_entry *locate_msr_entry(
+ struct vmx_msr_entry *start, struct vmx_msr_entry *end, uint32_t msr)
{
- const u32 *msr = key;
- const struct vmx_msr_entry *entry = elt;
+ while ( start < end )
+ {
+ struct vmx_msr_entry *mid = start + (end - start) / 2;
- if ( *msr > entry->index )
- return 1;
- if ( *msr < entry->index )
- return -1;
+ if ( msr < mid->index )
+ end = mid;
+ else if ( msr > mid->index )
+ start = mid + 1;
+ else
+ return mid;
+ }
- return 0;
+ return start;
}
-struct vmx_msr_entry *vmx_find_msr(u32 msr, int type)
+struct vmx_msr_entry *vmx_find_msr(const struct vcpu *v, uint32_t msr,
+ enum vmx_msr_list_type type)
{
- struct vcpu *curr = current;
- unsigned int msr_count;
- struct vmx_msr_entry *msr_area;
+ const struct arch_vmx_struct *vmx = &v->arch.hvm_vmx;
+ struct vmx_msr_entry *start = NULL, *ent, *end;
+ unsigned int substart = 0, subend = vmx->msr_save_count;
+ unsigned int total = vmx->msr_load_count;
- if ( type == VMX_GUEST_MSR )
- {
- msr_count = curr->arch.hvm_vmx.msr_count;
- msr_area = curr->arch.hvm_vmx.msr_area;
- }
- else
+ ASSERT(v == current || !vcpu_runnable(v));
+
+ switch ( type )
{
- ASSERT(type == VMX_HOST_MSR);
- msr_count = curr->arch.hvm_vmx.host_msr_count;
- msr_area = curr->arch.hvm_vmx.host_msr_area;
+ case VMX_MSR_HOST:
+ start = vmx->host_msr_area;
+ subend = vmx->host_msr_count;
+ total = subend;
+ break;
+
+ case VMX_MSR_GUEST:
+ start = vmx->msr_area;
+ break;
+
+ case VMX_MSR_GUEST_LOADONLY:
+ start = vmx->msr_area;
+ substart = subend;
+ subend = total;
+ break;
+
+ default:
+ ASSERT_UNREACHABLE();
}
- if ( msr_area == NULL )
+ if ( !start )
return NULL;
- return bsearch(&msr, msr_area, msr_count, sizeof(struct vmx_msr_entry),
- vmx_msr_entry_key_cmp);
+ end = start + total;
+ ent = locate_msr_entry(start + substart, start + subend, msr);
+
+ return ((ent < end) && (ent->index == msr)) ? ent : NULL;
}
-int vmx_read_guest_msr(u32 msr, u64 *val)
+int vmx_add_msr(struct vcpu *v, uint32_t msr, uint64_t val,
+ enum vmx_msr_list_type type)
{
- struct vmx_msr_entry *ent;
+ struct arch_vmx_struct *vmx = &v->arch.hvm_vmx;
+ struct vmx_msr_entry **ptr, *start = NULL, *ent, *end;
+ unsigned int substart, subend, total;
+ int rc;
- if ( (ent = vmx_find_msr(msr, VMX_GUEST_MSR)) != NULL )
+ ASSERT(v == current || !vcpu_runnable(v));
+
+ switch ( type )
{
- *val = ent->data;
- return 0;
- }
+ case VMX_MSR_HOST:
+ ptr = &vmx->host_msr_area;
+ substart = 0;
+ subend = vmx->host_msr_count;
+ total = subend;
+ break;
- return -ESRCH;
-}
+ case VMX_MSR_GUEST:
+ ptr = &vmx->msr_area;
+ substart = 0;
+ subend = vmx->msr_save_count;
+ total = vmx->msr_load_count;
+ break;
-int vmx_write_guest_msr(u32 msr, u64 val)
-{
- struct vmx_msr_entry *ent;
+ case VMX_MSR_GUEST_LOADONLY:
+ ptr = &vmx->msr_area;
+ substart = vmx->msr_save_count;
+ subend = vmx->msr_load_count;
+ total = subend;
+ break;
- if ( (ent = vmx_find_msr(msr, VMX_GUEST_MSR)) != NULL )
- {
- ent->data = val;
- return 0;
+ default:
+ ASSERT_UNREACHABLE();
+ return -EINVAL;
}
- return -ESRCH;
-}
-
-int vmx_add_msr(u32 msr, int type)
-{
- struct vcpu *curr = current;
- unsigned int idx, *msr_count;
- struct vmx_msr_entry **msr_area, *msr_area_elem;
+ vmx_vmcs_enter(v);
- if ( type == VMX_GUEST_MSR )
- {
- msr_count = &curr->arch.hvm_vmx.msr_count;
- msr_area = &curr->arch.hvm_vmx.msr_area;
- }
- else
+ /* Allocate memory on first use. */
+ if ( unlikely(!*ptr) )
{
- ASSERT(type == VMX_HOST_MSR);
- msr_count = &curr->arch.hvm_vmx.host_msr_count;
- msr_area = &curr->arch.hvm_vmx.host_msr_area;
- }
+ paddr_t addr;
- if ( *msr_area == NULL )
- {
- if ( (*msr_area = alloc_xenheap_page()) == NULL )
- return -ENOMEM;
+ if ( (*ptr = alloc_xenheap_page()) == NULL )
+ {
+ rc = -ENOMEM;
+ goto out;
+ }
- if ( type == VMX_GUEST_MSR )
+ addr = virt_to_maddr(*ptr);
+
+ switch ( type )
{
- __vmwrite(VM_EXIT_MSR_STORE_ADDR, virt_to_maddr(*msr_area));
- __vmwrite(VM_ENTRY_MSR_LOAD_ADDR, virt_to_maddr(*msr_area));
+ case VMX_MSR_HOST:
+ __vmwrite(VM_EXIT_MSR_LOAD_ADDR, addr);
+ break;
+
+ case VMX_MSR_GUEST:
+ case VMX_MSR_GUEST_LOADONLY:
+ __vmwrite(VM_EXIT_MSR_STORE_ADDR, addr);
+ __vmwrite(VM_ENTRY_MSR_LOAD_ADDR, addr);
+ break;
}
- else
- __vmwrite(VM_EXIT_MSR_LOAD_ADDR, virt_to_maddr(*msr_area));
}
- for ( idx = 0; idx < *msr_count && (*msr_area)[idx].index <= msr; idx++ )
- if ( (*msr_area)[idx].index == msr )
- return 0;
+ start = *ptr;
+ end = start + total;
+ ent = locate_msr_entry(start + substart, start + subend, msr);
- if ( *msr_count == (PAGE_SIZE / sizeof(struct vmx_msr_entry)) )
- return -ENOSPC;
+ if ( (ent < end) && (ent->index == msr) )
+ goto found;
- memmove(*msr_area + idx + 1, *msr_area + idx,
- sizeof(*msr_area_elem) * (*msr_count - idx));
+ /* If there isn't an existing entry for msr, insert room for one. */
+ if ( total == (PAGE_SIZE / sizeof(*ent)) )
+ {
+ rc = -ENOSPC;
+ goto out;
+ }
- msr_area_elem = *msr_area + idx;
- msr_area_elem->index = msr;
- msr_area_elem->mbz = 0;
+ memmove(ent + 1, ent, sizeof(*ent) * (end - ent));
- ++*msr_count;
+ ent->index = msr;
+ ent->mbz = 0;
- if ( type == VMX_GUEST_MSR )
+ switch ( type )
{
- msr_area_elem->data = 0;
- __vmwrite(VM_EXIT_MSR_STORE_COUNT, *msr_count);
- __vmwrite(VM_ENTRY_MSR_LOAD_COUNT, *msr_count);
- }
- else
- {
- rdmsrl(msr, msr_area_elem->data);
- __vmwrite(VM_EXIT_MSR_LOAD_COUNT, *msr_count);
+ case VMX_MSR_HOST:
+ __vmwrite(VM_EXIT_MSR_LOAD_COUNT, ++vmx->host_msr_count);
+ break;
+
+ case VMX_MSR_GUEST:
+ __vmwrite(VM_EXIT_MSR_STORE_COUNT, ++vmx->msr_save_count);
+
+ /* Fallthrough */
+ case VMX_MSR_GUEST_LOADONLY:
+ __vmwrite(VM_ENTRY_MSR_LOAD_COUNT, ++vmx->msr_load_count);
+ break;
}
- return 0;
+ /* Set the msr's value. */
+ found:
+ ent->data = val;
+ rc = 0;
+
+ out:
+ vmx_vmcs_exit(v);
+
+ return rc;
}
void vmx_set_eoi_exit_bitmap(struct vcpu *v, u8 vector)
diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c
index 610c8d6eb9..b0fababede 100644
--- a/xen/arch/x86/hvm/vmx/vmx.c
+++ b/xen/arch/x86/hvm/vmx/vmx.c
@@ -583,6 +583,12 @@ static void vmx_cpuid_policy_changed(struct vcpu *v)
vmx_clear_msr_intercept(v, MSR_PRED_CMD, VMX_MSR_RW);
else
vmx_set_msr_intercept(v, MSR_PRED_CMD, VMX_MSR_RW);
+
+ /* MSR_FLUSH_CMD is safe to pass through if the guest knows about it. */
+ if ( cp->feat.l1d_flush )
+ vmx_clear_msr_intercept(v, MSR_FLUSH_CMD, VMX_MSR_RW);
+ else
+ vmx_set_msr_intercept(v, MSR_FLUSH_CMD, VMX_MSR_RW);
}
int vmx_guest_x86_mode(struct vcpu *v)
@@ -2758,8 +2764,10 @@ enum
#define LBR_FROM_SIGNEXT_2MSB ((1ULL << 59) | (1ULL << 60))
-#define FIXUP_LBR_TSX (1u << 0)
-#define FIXUP_BDW_ERRATUM_BDF14 (1u << 1)
+#define LBR_MSRS_INSERTED (1u << 0)
+#define LBR_FIXUP_TSX (1u << 1)
+#define LBR_FIXUP_BDF14 (1u << 2)
+#define LBR_FIXUP_MASK (LBR_FIXUP_TSX | LBR_FIXUP_BDF14)
static bool __read_mostly lbr_tsx_fixup_needed;
static bool __read_mostly bdw_erratum_bdf14_fixup_needed;
@@ -2822,7 +2830,7 @@ static int is_last_branch_msr(u32 ecx)
static int vmx_msr_read_intercept(unsigned int msr, uint64_t *msr_content)
{
- const struct vcpu *curr = current;
+ struct vcpu *curr = current;
HVM_DBG_LOG(DBG_LEVEL_MSR, "ecx=%#x", msr);
@@ -2901,7 +2909,7 @@ static int vmx_msr_read_intercept(unsigned int msr, uint64_t *msr_content)
if ( passive_domain_do_rdmsr(msr, msr_content) )
goto done;
- if ( vmx_read_guest_msr(msr, msr_content) == 0 )
+ if ( vmx_read_guest_msr(curr, msr, msr_content) == 0 )
break;
if ( is_last_branch_msr(msr) )
@@ -3036,11 +3044,14 @@ void vmx_vlapic_msr_changed(struct vcpu *v)
static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content)
{
struct vcpu *v = current;
+ const struct cpuid_policy *cp = v->domain->arch.cpuid;
HVM_DBG_LOG(DBG_LEVEL_MSR, "ecx=%#x, msr_value=%#"PRIx64, msr, msr_content);
switch ( msr )
{
+ uint64_t rsvd;
+
case MSR_IA32_SYSENTER_CS:
__vmwrite(GUEST_SYSENTER_CS, msr_content);
break;
@@ -3093,45 +3104,85 @@ static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content)
wrmsrl(MSR_SYSCALL_MASK, msr_content);
break;
- case MSR_IA32_DEBUGCTLMSR: {
- int i, rc = 0;
- uint64_t supported = IA32_DEBUGCTLMSR_LBR | IA32_DEBUGCTLMSR_BTF;
+ case MSR_IA32_DEBUGCTLMSR:
+ rsvd = ~(IA32_DEBUGCTLMSR_LBR | IA32_DEBUGCTLMSR_BTF);
- if ( boot_cpu_has(X86_FEATURE_RTM) )
- supported |= IA32_DEBUGCTLMSR_RTM;
- if ( msr_content & ~supported )
+ /* TODO: Wire vPMU settings properly through the CPUID policy */
+ if ( vpmu_is_set(vcpu_vpmu(v), VPMU_CPU_HAS_BTS) )
{
- /* Perhaps some other bits are supported in vpmu. */
- if ( vpmu_do_wrmsr(msr, msr_content, supported) )
- break;
+ rsvd &= ~(IA32_DEBUGCTLMSR_TR | IA32_DEBUGCTLMSR_BTS |
+ IA32_DEBUGCTLMSR_BTINT);
+
+ if ( cpu_has(&current_cpu_data, X86_FEATURE_DSCPL) )
+ rsvd &= ~(IA32_DEBUGCTLMSR_BTS_OFF_OS |
+ IA32_DEBUGCTLMSR_BTS_OFF_USR);
}
- if ( msr_content & IA32_DEBUGCTLMSR_LBR )
+
+ if ( cp->feat.rtm )
+ rsvd &= ~IA32_DEBUGCTLMSR_RTM;
+
+ if ( msr_content & rsvd )
+ goto gp_fault;
+
+ /*
+ * When a guest first enables LBR, arrange to save and restore the LBR
+ * MSRs and allow the guest direct access.
+ *
+ * MSR_DEBUGCTL and LBR has existed almost as long as MSRs have
+ * existed, and there is no architectural way to hide the feature, or
+ * fail the attempt to enable LBR.
+ *
+ * Unknown host LBR MSRs or hitting -ENOSPC with the guest load/save
+ * list are definitely hypervisor bugs, whereas -ENOMEM for allocating
+ * the load/save list is simply unlucky (and shouldn't occur with
+ * sensible management by the toolstack).
+ *
+ * Either way, there is nothing we can do right now to recover, and
+ * the guest won't execute correctly either. Simply crash the domain
+ * to make the failure obvious.
+ */
+ if ( !(v->arch.hvm_vmx.lbr_flags & LBR_MSRS_INSERTED) &&
+ (msr_content & IA32_DEBUGCTLMSR_LBR) )
{
const struct lbr_info *lbr = last_branch_msr_get();
- if ( lbr == NULL )
- break;
- for ( ; (rc == 0) && lbr->count; lbr++ )
- for ( i = 0; (rc == 0) && (i < lbr->count); i++ )
- if ( (rc = vmx_add_guest_msr(lbr->base + i)) == 0 )
+ if ( unlikely(!lbr) )
+ {
+ gprintk(XENLOG_ERR, "Unknown Host LBR MSRs\n");
+ domain_crash(v->domain);
+ return X86EMUL_OKAY;
+ }
+
+ for ( ; lbr->count; lbr++ )
+ {
+ unsigned int i;
+
+ for ( i = 0; i < lbr->count; i++ )
+ {
+ int rc = vmx_add_guest_msr(v, lbr->base + i, 0);
+
+ if ( unlikely(rc) )
{
- vmx_clear_msr_intercept(v, lbr->base + i, VMX_MSR_RW);
- if ( lbr_tsx_fixup_needed )
- v->arch.hvm_vmx.lbr_fixup_enabled |= FIXUP_LBR_TSX;
- if ( bdw_erratum_bdf14_fixup_needed )
- v->arch.hvm_vmx.lbr_fixup_enabled |=
- FIXUP_BDW_ERRATUM_BDF14;
+ gprintk(XENLOG_ERR,
+ "Guest load/save list error %d\n", rc);
+ domain_crash(v->domain);
+ return X86EMUL_OKAY;
}
- }
- if ( (rc < 0) ||
- (msr_content && (vmx_add_host_load_msr(msr) < 0)) )
- hvm_inject_hw_exception(TRAP_machine_check, X86_EVENT_NO_EC);
- else
- __vmwrite(GUEST_IA32_DEBUGCTL, msr_content);
+ vmx_clear_msr_intercept(v, lbr->base + i, VMX_MSR_RW);
+ }
+ }
+
+ v->arch.hvm_vmx.lbr_flags |= LBR_MSRS_INSERTED;
+ if ( lbr_tsx_fixup_needed )
+ v->arch.hvm_vmx.lbr_flags |= LBR_FIXUP_TSX;
+ if ( bdw_erratum_bdf14_fixup_needed )
+ v->arch.hvm_vmx.lbr_flags |= LBR_FIXUP_BDF14;
+ }
+ __vmwrite(GUEST_IA32_DEBUGCTL, msr_content);
break;
- }
+
case MSR_IA32_FEATURE_CONTROL:
case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
/* None of these MSRs are writeable. */
@@ -3154,7 +3205,7 @@ static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content)
if ( wrmsr_viridian_regs(msr, msr_content) )
break;
- if ( vmx_write_guest_msr(msr, msr_content) == 0 ||
+ if ( vmx_write_guest_msr(v, msr, msr_content) == 0 ||
is_last_branch_msr(msr) )
break;
@@ -3701,6 +3752,7 @@ void vmx_vmexit_handler(struct cpu_user_regs *regs)
*/
__vmread(EXIT_QUALIFICATION, &exit_qualification);
HVMTRACE_1D(TRAP_DEBUG, exit_qualification);
+ __restore_debug_registers(v);
write_debugreg(6, exit_qualification | DR_STATUS_RESERVED_ONE);
if ( !v->domain->debugger_attached )
{
@@ -4165,11 +4217,11 @@ out:
static void lbr_tsx_fixup(void)
{
struct vcpu *curr = current;
- unsigned int msr_count = curr->arch.hvm_vmx.msr_count;
+ unsigned int msr_count = curr->arch.hvm_vmx.msr_save_count;
struct vmx_msr_entry *msr_area = curr->arch.hvm_vmx.msr_area;
struct vmx_msr_entry *msr;
- if ( (msr = vmx_find_msr(lbr_from_start, VMX_GUEST_MSR)) != NULL )
+ if ( (msr = vmx_find_msr(curr, lbr_from_start, VMX_MSR_GUEST)) != NULL )
{
/*
* Sign extend into bits 61:62 while preserving bit 63
@@ -4179,15 +4231,15 @@ static void lbr_tsx_fixup(void)
msr->data |= ((LBR_FROM_SIGNEXT_2MSB & msr->data) << 2);
}
- if ( (msr = vmx_find_msr(lbr_lastint_from, VMX_GUEST_MSR)) != NULL )
+ if ( (msr = vmx_find_msr(curr, lbr_lastint_from, VMX_MSR_GUEST)) != NULL )
msr->data |= ((LBR_FROM_SIGNEXT_2MSB & msr->data) << 2);
}
-static void sign_extend_msr(u32 msr, int type)
+static void sign_extend_msr(struct vcpu *v, u32 msr, int type)
{
struct vmx_msr_entry *entry;
- if ( (entry = vmx_find_msr(msr, type)) != NULL )
+ if ( (entry = vmx_find_msr(v, msr, type)) != NULL )
{
if ( entry->data & VADDR_TOP_BIT )
entry->data |= CANONICAL_MASK;
@@ -4198,6 +4250,8 @@ static void sign_extend_msr(u32 msr, int type)
static void bdw_erratum_bdf14_fixup(void)
{
+ struct vcpu *curr = current;
+
/*
* Occasionally, on certain Broadwell CPUs MSR_IA32_LASTINTTOIP has
* been observed to have the top three bits corrupted as though the
@@ -4207,17 +4261,17 @@ static void bdw_erratum_bdf14_fixup(void)
* erratum BDF14. Fix up MSR_IA32_LASTINT{FROM,TO}IP by
* sign-extending into bits 48:63.
*/
- sign_extend_msr(MSR_IA32_LASTINTFROMIP, VMX_GUEST_MSR);
- sign_extend_msr(MSR_IA32_LASTINTTOIP, VMX_GUEST_MSR);
+ sign_extend_msr(curr, MSR_IA32_LASTINTFROMIP, VMX_MSR_GUEST);
+ sign_extend_msr(curr, MSR_IA32_LASTINTTOIP, VMX_MSR_GUEST);
}
static void lbr_fixup(void)
{
struct vcpu *curr = current;
- if ( curr->arch.hvm_vmx.lbr_fixup_enabled & FIXUP_LBR_TSX )
+ if ( curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_TSX )
lbr_tsx_fixup();
- if ( curr->arch.hvm_vmx.lbr_fixup_enabled & FIXUP_BDW_ERRATUM_BDF14 )
+ if ( curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_BDF14 )
bdw_erratum_bdf14_fixup();
}
@@ -4285,7 +4339,7 @@ bool vmx_vmenter_helper(const struct cpu_user_regs *regs)
}
out:
- if ( unlikely(curr->arch.hvm_vmx.lbr_fixup_enabled) )
+ if ( unlikely(curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_MASK) )
lbr_fixup();
HVMTRACE_ND(VMENTRY, 0, 1/*cycles*/, 0, 0, 0, 0, 0, 0, 0);
diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
index bcf46c0743..7d4871b791 100644
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -613,6 +613,9 @@ static int alloc_segdesc_page(struct page_info *page)
return i == 512 ? 0 : -EINVAL;
}
+static int _get_page_type(struct page_info *page, unsigned long type,
+ bool preemptible);
+
static int get_page_and_type_from_mfn(
mfn_t mfn, unsigned long type, struct domain *d,
int partial, int preemptible)
@@ -624,9 +627,7 @@ static int get_page_and_type_from_mfn(
unlikely(!get_page_from_mfn(mfn, d)) )
return -EINVAL;
- rc = (preemptible ?
- get_page_type_preemptible(page, type) :
- (get_page_type(page, type) ? 0 : -EINVAL));
+ rc = _get_page_type(page, type, preemptible);
if ( unlikely(rc) && partial >= 0 &&
(!preemptible || page != current->arch.old_guest_table) )
@@ -1115,7 +1116,7 @@ get_page_from_l2e(
int rc;
if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) )
- return 1;
+ return pv_l1tf_check_l2e(d, l2e) ? -ERESTART : 1;
if ( unlikely((l2e_get_flags(l2e) & L2_DISALLOW_MASK)) )
{
@@ -1146,7 +1147,7 @@ get_page_from_l3e(
int rc;
if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
- return 1;
+ return pv_l1tf_check_l3e(d, l3e) ? -ERESTART : 1;
if ( unlikely((l3e_get_flags(l3e) & l3_disallow_mask(d))) )
{
@@ -1179,7 +1180,7 @@ get_page_from_l4e(
int rc;
if ( !(l4e_get_flags(l4e) & _PAGE_PRESENT) )
- return 1;
+ return pv_l1tf_check_l4e(d, l4e) ? -ERESTART : 1;
if ( unlikely((l4e_get_flags(l4e) & L4_DISALLOW_MASK)) )
{
@@ -1389,6 +1390,13 @@ static int alloc_l1_table(struct page_info *page)
for ( i = 0; i < L1_PAGETABLE_ENTRIES; i++ )
{
+ if ( !(l1e_get_flags(pl1e[i]) & _PAGE_PRESENT) )
+ {
+ ret = pv_l1tf_check_l1e(d, pl1e[i]) ? -ERESTART : 0;
+ if ( ret )
+ goto out;
+ }
+
switch ( ret = get_page_from_l1e(pl1e[i], d, d) )
{
default:
@@ -1409,6 +1417,7 @@ static int alloc_l1_table(struct page_info *page)
fail:
gdprintk(XENLOG_WARNING, "Failure in alloc_l1_table: slot %#x\n", i);
+ out:
while ( i-- > 0 )
put_page_from_l1e(pl1e[i], d);
@@ -1456,8 +1465,7 @@ static int create_pae_xen_mappings(struct domain *d, l3_pgentry_t *pl3e)
return 1;
}
-static int alloc_l2_table(struct page_info *page, unsigned long type,
- int preemptible)
+static int alloc_l2_table(struct page_info *page, unsigned long type)
{
struct domain *d = page_get_owner(page);
unsigned long pfn = mfn_x(page_to_mfn(page));
@@ -1469,8 +1477,7 @@ static int alloc_l2_table(struct page_info *page, unsigned long type,
for ( i = page->nr_validated_ptes; i < L2_PAGETABLE_ENTRIES; i++ )
{
- if ( preemptible && i > page->nr_validated_ptes
- && hypercall_preempt_check() )
+ if ( i > page->nr_validated_ptes && hypercall_preempt_check() )
{
page->nr_validated_ptes = i;
rc = -ERESTART;
@@ -1481,6 +1488,12 @@ static int alloc_l2_table(struct page_info *page, unsigned long type,
(rc = get_page_from_l2e(pl2e[i], pfn, d)) > 0 )
continue;
+ if ( unlikely(rc == -ERESTART) )
+ {
+ page->nr_validated_ptes = i;
+ break;
+ }
+
if ( rc < 0 )
{
gdprintk(XENLOG_WARNING, "Failure in alloc_l2_table: slot %#x\n", i);
@@ -1763,7 +1776,7 @@ static void free_l1_table(struct page_info *page)
}
-static int free_l2_table(struct page_info *page, int preemptible)
+static int free_l2_table(struct page_info *page)
{
struct domain *d = page_get_owner(page);
unsigned long pfn = mfn_x(page_to_mfn(page));
@@ -1777,7 +1790,7 @@ static int free_l2_table(struct page_info *page, int preemptible)
do {
if ( is_guest_l2_slot(d, page->u.inuse.type_info, i) &&
put_page_from_l2e(pl2e[i], pfn) == 0 &&
- preemptible && i && hypercall_preempt_check() )
+ i && hypercall_preempt_check() )
{
page->nr_validated_ptes = i;
err = -ERESTART;
@@ -2055,6 +2068,8 @@ static int mod_l1_entry(l1_pgentry_t *pl1e, l1_pgentry_t nl1e,
rc = -EBUSY;
}
}
+ else if ( pv_l1tf_check_l1e(pt_dom, nl1e) )
+ return -ERESTART;
else if ( unlikely(!UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, pt_vcpu,
preserve_ad)) )
{
@@ -2118,6 +2133,8 @@ static int mod_l2_entry(l2_pgentry_t *pl2e,
rc = -EBUSY;
}
}
+ else if ( pv_l1tf_check_l2e(d, nl2e) )
+ return -ERESTART;
else if ( unlikely(!UPDATE_ENTRY(l2, pl2e, ol2e, nl2e, pfn, vcpu,
preserve_ad)) )
{
@@ -2179,6 +2196,8 @@ static int mod_l3_entry(l3_pgentry_t *pl3e,
rc = -EFAULT;
}
}
+ else if ( pv_l1tf_check_l3e(d, nl3e) )
+ return -ERESTART;
else if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn, vcpu,
preserve_ad)) )
{
@@ -2244,6 +2263,8 @@ static int mod_l4_entry(l4_pgentry_t *pl4e,
rc = -EFAULT;
}
}
+ else if ( pv_l1tf_check_l4e(d, nl4e) )
+ return -ERESTART;
else if ( unlikely(!UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn, vcpu,
preserve_ad)) )
{
@@ -2373,7 +2394,8 @@ static int alloc_page_type(struct page_info *page, unsigned long type,
rc = alloc_l1_table(page);
break;
case PGT_l2_page_table:
- rc = alloc_l2_table(page, type, preemptible);
+ ASSERT(preemptible);
+ rc = alloc_l2_table(page, type);
break;
case PGT_l3_page_table:
ASSERT(preemptible);
@@ -2463,7 +2485,8 @@ int free_page_type(struct page_info *page, unsigned long type,
rc = 0;
break;
case PGT_l2_page_table:
- rc = free_l2_table(page, preemptible);
+ ASSERT(preemptible);
+ rc = free_l2_table(page);
break;
case PGT_l3_page_table:
ASSERT(preemptible);
@@ -3550,12 +3573,9 @@ long do_mmuext_op(
}
if ( rc == -ERESTART )
- {
- ASSERT(i < count);
rc = hypercall_create_continuation(
__HYPERVISOR_mmuext_op, "hihi",
uops, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom);
- }
else if ( curr->arch.old_guest_table )
{
XEN_GUEST_HANDLE_PARAM(void) null;
@@ -3861,12 +3881,9 @@ long do_mmu_update(
}
if ( rc == -ERESTART )
- {
- ASSERT(i < count);
rc = hypercall_create_continuation(
__HYPERVISOR_mmu_update, "hihi",
ureqs, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom);
- }
else if ( curr->arch.old_guest_table )
{
XEN_GUEST_HANDLE_PARAM(void) null;
@@ -4121,7 +4138,13 @@ static int __do_update_va_mapping(
long do_update_va_mapping(unsigned long va, u64 val64,
unsigned long flags)
{
- return __do_update_va_mapping(va, val64, flags, current->domain);
+ int rc = __do_update_va_mapping(va, val64, flags, current->domain);
+
+ if ( rc == -ERESTART )
+ rc = hypercall_create_continuation(
+ __HYPERVISOR_update_va_mapping, "lll", va, val64, flags);
+
+ return rc;
}
long do_update_va_mapping_otherdomain(unsigned long va, u64 val64,
@@ -4138,6 +4161,46 @@ long do_update_va_mapping_otherdomain(unsigned long va, u64 val64,
put_pg_owner(pg_owner);
+ if ( rc == -ERESTART )
+ rc = hypercall_create_continuation(
+ __HYPERVISOR_update_va_mapping_otherdomain,
+ "llli", va, val64, flags, domid);
+
+ return rc;
+}
+
+int compat_update_va_mapping(unsigned int va, uint32_t lo, uint32_t hi,
+ unsigned int flags)
+{
+ int rc = __do_update_va_mapping(va, ((uint64_t)hi << 32) | lo,
+ flags, current->domain);
+
+ if ( rc == -ERESTART )
+ rc = hypercall_create_continuation(
+ __HYPERVISOR_update_va_mapping, "iiii", va, lo, hi, flags);
+
+ return rc;
+}
+
+int compat_update_va_mapping_otherdomain(unsigned int va,
+ uint32_t lo, uint32_t hi,
+ unsigned int flags, domid_t domid)
+{
+ struct domain *pg_owner;
+ int rc;
+
+ if ( (pg_owner = get_pg_owner(domid)) == NULL )
+ return -ESRCH;
+
+ rc = __do_update_va_mapping(va, ((uint64_t)hi << 32) | lo, flags, pg_owner);
+
+ put_pg_owner(pg_owner);
+
+ if ( rc == -ERESTART )
+ rc = hypercall_create_continuation(
+ __HYPERVISOR_update_va_mapping_otherdomain,
+ "iiiii", va, lo, hi, flags, domid);
+
return rc;
}
diff --git a/xen/arch/x86/mm/paging.c b/xen/arch/x86/mm/paging.c
index 2b0445ffe9..dcee496eb0 100644
--- a/xen/arch/x86/mm/paging.c
+++ b/xen/arch/x86/mm/paging.c
@@ -873,6 +873,8 @@ void paging_dump_domain_info(struct domain *d)
printk(" paging assistance: ");
if ( paging_mode_shadow(d) )
printk("shadow ");
+ if ( paging_mode_sh_forced(d) )
+ printk("forced ");
if ( paging_mode_hap(d) )
printk("hap ");
if ( paging_mode_refcounts(d) )
diff --git a/xen/arch/x86/mm/shadow/common.c b/xen/arch/x86/mm/shadow/common.c
index dd61b50eb7..fd42d734e7 100644
--- a/xen/arch/x86/mm/shadow/common.c
+++ b/xen/arch/x86/mm/shadow/common.c
@@ -3177,6 +3177,15 @@ static void sh_new_mode(struct domain *d, u32 new_mode)
ASSERT(paging_locked_by_me(d));
ASSERT(d != current->domain);
+ /*
+ * If PG_SH_forced has previously been activated because of writing an
+ * L1TF-vulnerable PTE, it must remain active for the remaining lifetime
+ * of the domain, even if the logdirty mode needs to be controlled for
+ * migration purposes.
+ */
+ if ( paging_mode_sh_forced(d) )
+ new_mode |= PG_SH_forced | PG_SH_enable;
+
d->arch.paging.mode = new_mode;
for_each_vcpu(d, v)
sh_update_paging_modes(v);
@@ -4057,6 +4066,33 @@ void shadow_audit_tables(struct vcpu *v)
#endif /* Shadow audit */
+#ifdef CONFIG_PV
+
+void pv_l1tf_tasklet(unsigned long data)
+{
+ struct domain *d = (void *)data;
+
+ domain_pause(d);
+ paging_lock(d);
+
+ if ( !paging_mode_sh_forced(d) && !d->is_dying )
+ {
+ int ret = shadow_one_bit_enable(d, PG_SH_forced);
+
+ if ( ret )
+ {
+ printk(XENLOG_G_ERR "d%d Failed to enable PG_SH_forced: %d\n",
+ d->domain_id, ret);
+ domain_crash(d);
+ }
+ }
+
+ paging_unlock(d);
+ domain_unpause(d);
+}
+
+#endif /* CONFIG_PV */
+
/*
* Local variables:
* mode: C
diff --git a/xen/arch/x86/mpparse.c b/xen/arch/x86/mpparse.c
index 49140e46f0..f3f6d48668 100644
--- a/xen/arch/x86/mpparse.c
+++ b/xen/arch/x86/mpparse.c
@@ -68,19 +68,26 @@ physid_mask_t phys_cpu_present_map;
void __init set_nr_cpu_ids(unsigned int max_cpus)
{
+ unsigned int tot_cpus = num_processors + disabled_cpus;
+
if (!max_cpus)
- max_cpus = num_processors + disabled_cpus;
+ max_cpus = tot_cpus;
if (max_cpus > NR_CPUS)
max_cpus = NR_CPUS;
else if (!max_cpus)
max_cpus = 1;
printk(XENLOG_INFO "SMP: Allowing %u CPUs (%d hotplug CPUs)\n",
max_cpus, max_t(int, max_cpus - num_processors, 0));
- nr_cpu_ids = max_cpus;
+
+ if (!park_offline_cpus)
+ tot_cpus = max_cpus;
+ nr_cpu_ids = min(tot_cpus, NR_CPUS + 0u);
+ if (park_offline_cpus && nr_cpu_ids < num_processors)
+ printk(XENLOG_WARNING "SMP: Cannot bring up %u further CPUs\n",
+ num_processors - nr_cpu_ids);
#ifndef nr_cpumask_bits
- nr_cpumask_bits = (max_cpus + (BITS_PER_LONG - 1)) &
- ~(BITS_PER_LONG - 1);
+ nr_cpumask_bits = ROUNDUP(nr_cpu_ids, BITS_PER_LONG);
printk(XENLOG_DEBUG "NR_CPUS:%u nr_cpumask_bits:%u\n",
NR_CPUS, nr_cpumask_bits);
#endif
diff --git a/xen/arch/x86/msr.c b/xen/arch/x86/msr.c
index 1e12ccb729..1a591dd2b5 100644
--- a/xen/arch/x86/msr.c
+++ b/xen/arch/x86/msr.c
@@ -150,6 +150,7 @@ int guest_rdmsr(const struct vcpu *v, uint32_t msr, uint64_t *val)
case MSR_AMD_PATCHLOADER:
case MSR_IA32_UCODE_WRITE:
case MSR_PRED_CMD:
+ case MSR_FLUSH_CMD:
/* Write-only */
goto gp_fault;
@@ -254,6 +255,17 @@ int guest_wrmsr(struct vcpu *v, uint32_t msr, uint64_t val)
wrmsrl(MSR_PRED_CMD, val);
break;
+ case MSR_FLUSH_CMD:
+ if ( !cp->feat.l1d_flush )
+ goto gp_fault; /* MSR available? */
+
+ if ( val & ~FLUSH_CMD_L1D )
+ goto gp_fault; /* Rsvd bit set? */
+
+ if ( v == curr )
+ wrmsrl(MSR_FLUSH_CMD, val);
+ break;
+
case MSR_INTEL_MISC_FEATURES_ENABLES:
{
bool old_cpuid_faulting = vp->misc_features_enables.cpuid_faulting;
diff --git a/xen/arch/x86/oprofile/nmi_int.c b/xen/arch/x86/oprofile/nmi_int.c
index d8f5230906..3dfb8fef93 100644
--- a/xen/arch/x86/oprofile/nmi_int.c
+++ b/xen/arch/x86/oprofile/nmi_int.c
@@ -182,7 +182,7 @@ int nmi_reserve_counters(void)
if (!allocate_msrs())
return -ENOMEM;
- /* We walk a thin line between law and rape here.
+ /*
* We need to be careful to install our NMI handler
* without actually triggering any NMIs as this will
* break the core code horrifically.
diff --git a/xen/arch/x86/percpu.c b/xen/arch/x86/percpu.c
index c9997b7937..8be4ebddf4 100644
--- a/xen/arch/x86/percpu.c
+++ b/xen/arch/x86/percpu.c
@@ -28,7 +28,7 @@ static int init_percpu_area(unsigned int cpu)
char *p;
if ( __per_cpu_offset[cpu] != INVALID_PERCPU_AREA )
- return -EBUSY;
+ return 0;
if ( (p = alloc_xenheap_pages(PERCPU_ORDER, 0)) == NULL )
return -ENOMEM;
@@ -76,9 +76,12 @@ static int cpu_percpu_callback(
break;
case CPU_UP_CANCELED:
case CPU_DEAD:
- free_percpu_area(cpu);
+ if ( !park_offline_cpus )
+ free_percpu_area(cpu);
break;
- default:
+ case CPU_REMOVE:
+ if ( park_offline_cpus )
+ free_percpu_area(cpu);
break;
}
diff --git a/xen/arch/x86/pv/domain.c b/xen/arch/x86/pv/domain.c
index a4f0bd239d..3230ac6a22 100644
--- a/xen/arch/x86/pv/domain.c
+++ b/xen/arch/x86/pv/domain.c
@@ -13,6 +13,7 @@
#include <asm/invpcid.h>
#include <asm/spec_ctrl.h>
#include <asm/pv/domain.h>
+#include <asm/shadow.h>
static __read_mostly enum {
PCID_OFF,
@@ -209,6 +210,8 @@ int pv_vcpu_initialise(struct vcpu *v)
void pv_domain_destroy(struct domain *d)
{
+ pv_l1tf_domain_destroy(d);
+
destroy_perdomain_mapping(d, GDT_LDT_VIRT_START,
GDT_LDT_MBYTES << (20 - PAGE_SHIFT));
@@ -229,6 +232,8 @@ int pv_domain_initialise(struct domain *d)
};
int rc = -ENOMEM;
+ pv_l1tf_domain_init(d);
+
d->arch.pv_domain.gdt_ldt_l1tab =
alloc_xenheap_pages(0, MEMF_node(domain_to_node(d)));
if ( !d->arch.pv_domain.gdt_ldt_l1tab )
diff --git a/xen/arch/x86/pv/ro-page-fault.c b/xen/arch/x86/pv/ro-page-fault.c
index aa8d5a7556..a3c0c2dd19 100644
--- a/xen/arch/x86/pv/ro-page-fault.c
+++ b/xen/arch/x86/pv/ro-page-fault.c
@@ -29,6 +29,7 @@
#include <asm/mm.h>
#include <asm/pci.h>
#include <asm/pv/mm.h>
+#include <asm/shadow.h>
#include "emulate.h"
#include "mm.h"
@@ -129,6 +130,10 @@ static int ptwr_emulated_update(unsigned long addr, intpte_t *p_old,
/* Check the new PTE. */
nl1e = l1e_from_intpte(val);
+
+ if ( !(l1e_get_flags(nl1e) & _PAGE_PRESENT) && pv_l1tf_check_l1e(d, nl1e) )
+ return X86EMUL_RETRY;
+
switch ( ret = get_page_from_l1e(nl1e, d, d) )
{
default:
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index a3172ca92c..3cd3e81b30 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -62,6 +62,9 @@ boolean_param("nosmp", opt_nosmp);
static unsigned int __initdata max_cpus;
integer_param("maxcpus", max_cpus);
+int8_t __read_mostly opt_smt = -1;
+boolean_param("smt", opt_smt);
+
/* opt_invpcid: If false, don't use INVPCID instruction even if available. */
static bool __initdata opt_invpcid = true;
boolean_param("invpcid", opt_invpcid);
@@ -665,7 +668,7 @@ void __init noreturn __start_xen(unsigned long mbi_p)
{
char *memmap_type = NULL;
char *cmdline, *kextra, *loader;
- unsigned int initrdidx;
+ unsigned int initrdidx, num_parked = 0;
multiboot_info_t *mbi;
module_t *mod;
unsigned long nr_pages, raw_max_page, modules_headroom, *module_map;
@@ -909,6 +912,18 @@ void __init noreturn __start_xen(unsigned long mbi_p)
/* Sanitise the raw E820 map to produce a final clean version. */
max_page = raw_max_page = init_e820(memmap_type, &e820_raw);
+ if ( !efi_enabled(EFI_BOOT) )
+ {
+ /*
+ * Supplement the heuristics in l1tf_calculations() by assuming that
+ * anything referenced in the E820 may be cacheable.
+ */
+ l1tf_safe_maddr =
+ max(l1tf_safe_maddr,
+ ROUNDUP(e820_raw.map[e820_raw.nr_map - 1].addr +
+ e820_raw.map[e820_raw.nr_map - 1].size, PAGE_SIZE));
+ }
+
/* Create a temporary copy of the E820 map. */
memcpy(&boot_e820, &e820, sizeof(e820));
@@ -1494,7 +1509,8 @@ void __init noreturn __start_xen(unsigned long mbi_p)
else
{
set_nr_cpu_ids(max_cpus);
- max_cpus = nr_cpu_ids;
+ if ( !max_cpus )
+ max_cpus = nr_cpu_ids;
}
if ( xen_guest )
@@ -1617,16 +1633,30 @@ void __init noreturn __start_xen(unsigned long mbi_p)
/* Set up node_to_cpumask based on cpu_to_node[]. */
numa_add_cpu(i);
- if ( (num_online_cpus() < max_cpus) && !cpu_online(i) )
+ if ( (park_offline_cpus || num_online_cpus() < max_cpus) &&
+ !cpu_online(i) )
{
int ret = cpu_up(i);
if ( ret != 0 )
printk("Failed to bring up CPU %u (error %d)\n", i, ret);
+ else if ( num_online_cpus() > max_cpus ||
+ (!opt_smt &&
+ cpu_data[i].compute_unit_id == INVALID_CUID &&
+ cpumask_weight(per_cpu(cpu_sibling_mask, i)) > 1) )
+ {
+ ret = cpu_down(i);
+ if ( !ret )
+ ++num_parked;
+ else
+ printk("Could not re-offline CPU%u (%d)\n", i, ret);
+ }
}
}
}
printk("Brought up %ld CPUs\n", (long)num_online_cpus());
+ if ( num_parked )
+ printk(XENLOG_INFO "Parked %u CPUs\n", num_parked);
smp_cpus_done();
do_initcalls();
diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index d4478e6132..7e76cc3d68 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -63,6 +63,8 @@ static cpumask_t scratch_cpu0mask;
cpumask_t cpu_online_map __read_mostly;
EXPORT_SYMBOL(cpu_online_map);
+bool __read_mostly park_offline_cpus;
+
unsigned int __read_mostly nr_sockets;
cpumask_t **__read_mostly socket_cpumask;
static cpumask_t *secondary_socket_cpumask;
@@ -234,33 +236,41 @@ static void link_thread_siblings(int cpu1, int cpu2)
cpumask_set_cpu(cpu2, per_cpu(cpu_core_mask, cpu1));
}
-static void set_cpu_sibling_map(int cpu)
+static void set_cpu_sibling_map(unsigned int cpu)
{
- int i;
+ unsigned int i;
struct cpuinfo_x86 *c = cpu_data;
cpumask_set_cpu(cpu, &cpu_sibling_setup_map);
cpumask_set_cpu(cpu, socket_cpumask[cpu_to_socket(cpu)]);
+ cpumask_set_cpu(cpu, per_cpu(cpu_core_mask, cpu));
+ cpumask_set_cpu(cpu, per_cpu(cpu_sibling_mask, cpu));
if ( c[cpu].x86_num_siblings > 1 )
{
for_each_cpu ( i, &cpu_sibling_setup_map )
{
- if ( cpu_has(c, X86_FEATURE_TOPOEXT) ) {
- if ( (c[cpu].phys_proc_id == c[i].phys_proc_id) &&
- (c[cpu].compute_unit_id == c[i].compute_unit_id) )
+ if ( cpu == i || c[cpu].phys_proc_id != c[i].phys_proc_id )
+ continue;
+ if ( c[cpu].compute_unit_id != INVALID_CUID &&
+ c[i].compute_unit_id != INVALID_CUID )
+ {
+ if ( c[cpu].compute_unit_id == c[i].compute_unit_id )
link_thread_siblings(cpu, i);
- } else if ( (c[cpu].phys_proc_id == c[i].phys_proc_id) &&
- (c[cpu].cpu_core_id == c[i].cpu_core_id) ) {
- link_thread_siblings(cpu, i);
}
+ else if ( c[cpu].cpu_core_id != XEN_INVALID_CORE_ID &&
+ c[i].cpu_core_id != XEN_INVALID_CORE_ID )
+ {
+ if ( c[cpu].cpu_core_id == c[i].cpu_core_id )
+ link_thread_siblings(cpu, i);
+ }
+ else
+ printk(XENLOG_WARNING
+ "CPU%u: unclear relationship with CPU%u\n",
+ cpu, i);
}
}
- else
- {
- cpumask_set_cpu(cpu, per_cpu(cpu_sibling_mask, cpu));
- }
if ( c[cpu].x86_max_cores == 1 )
{
@@ -887,7 +897,14 @@ static void cleanup_cpu_root_pgt(unsigned int cpu)
}
}
-static void cpu_smpboot_free(unsigned int cpu)
+/*
+ * The 'remove' boolean controls whether a CPU is just getting offlined (and
+ * parked), or outright removed / offlined without parking. Parked CPUs need
+ * things like their stack, GDT, IDT, TSS, and per-CPU data still available.
+ * A few other items, in particular CPU masks, are also retained, as it's
+ * difficult to prove that they're entirely unreferenced from parked CPUs.
+ */
+static void cpu_smpboot_free(unsigned int cpu, bool remove)
{
unsigned int order, socket = cpu_to_socket(cpu);
struct cpuinfo_x86 *c = cpu_data;
@@ -898,15 +915,19 @@ static void cpu_smpboot_free(unsigned int cpu)
socket_cpumask[socket] = NULL;
}
- c[cpu].phys_proc_id = XEN_INVALID_SOCKET_ID;
- c[cpu].cpu_core_id = XEN_INVALID_CORE_ID;
- c[cpu].compute_unit_id = INVALID_CUID;
cpumask_clear_cpu(cpu, &cpu_sibling_setup_map);
- free_cpumask_var(per_cpu(cpu_sibling_mask, cpu));
- free_cpumask_var(per_cpu(cpu_core_mask, cpu));
- if ( per_cpu(scratch_cpumask, cpu) != &scratch_cpu0mask )
- free_cpumask_var(per_cpu(scratch_cpumask, cpu));
+ if ( remove )
+ {
+ c[cpu].phys_proc_id = XEN_INVALID_SOCKET_ID;
+ c[cpu].cpu_core_id = XEN_INVALID_CORE_ID;
+ c[cpu].compute_unit_id = INVALID_CUID;
+
+ FREE_CPUMASK_VAR(per_cpu(cpu_sibling_mask, cpu));
+ FREE_CPUMASK_VAR(per_cpu(cpu_core_mask, cpu));
+ if ( per_cpu(scratch_cpumask, cpu) != &scratch_cpu0mask )
+ FREE_CPUMASK_VAR(per_cpu(scratch_cpumask, cpu));
+ }
cleanup_cpu_root_pgt(cpu);
@@ -928,19 +949,21 @@ static void cpu_smpboot_free(unsigned int cpu)
}
order = get_order_from_pages(NR_RESERVED_GDT_PAGES);
- free_xenheap_pages(per_cpu(gdt_table, cpu), order);
+ if ( remove )
+ FREE_XENHEAP_PAGES(per_cpu(gdt_table, cpu), order);
free_xenheap_pages(per_cpu(compat_gdt_table, cpu), order);
- order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t));
- free_xenheap_pages(idt_tables[cpu], order);
- idt_tables[cpu] = NULL;
-
- if ( stack_base[cpu] != NULL )
+ if ( remove )
{
- memguard_unguard_stack(stack_base[cpu]);
- free_xenheap_pages(stack_base[cpu], STACK_ORDER);
- stack_base[cpu] = NULL;
+ order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t));
+ FREE_XENHEAP_PAGES(idt_tables[cpu], order);
+
+ if ( stack_base[cpu] )
+ {
+ memguard_unguard_stack(stack_base[cpu]);
+ FREE_XENHEAP_PAGES(stack_base[cpu], STACK_ORDER);
+ }
}
}
@@ -955,15 +978,17 @@ static int cpu_smpboot_alloc(unsigned int cpu)
if ( node != NUMA_NO_NODE )
memflags = MEMF_node(node);
- stack_base[cpu] = alloc_xenheap_pages(STACK_ORDER, memflags);
+ if ( stack_base[cpu] == NULL )
+ stack_base[cpu] = alloc_xenheap_pages(STACK_ORDER, memflags);
if ( stack_base[cpu] == NULL )
goto out;
memguard_guard_stack(stack_base[cpu]);
order = get_order_from_pages(NR_RESERVED_GDT_PAGES);
- per_cpu(gdt_table, cpu) = gdt = alloc_xenheap_pages(order, memflags);
+ gdt = per_cpu(gdt_table, cpu) ?: alloc_xenheap_pages(order, memflags);
if ( gdt == NULL )
goto out;
+ per_cpu(gdt_table, cpu) = gdt;
memcpy(gdt, boot_cpu_gdt_table, NR_RESERVED_GDT_PAGES * PAGE_SIZE);
BUILD_BUG_ON(NR_CPUS > 0x10000);
gdt[PER_CPU_GDT_ENTRY - FIRST_RESERVED_GDT_ENTRY].a = cpu;
@@ -975,7 +1000,8 @@ static int cpu_smpboot_alloc(unsigned int cpu)
gdt[PER_CPU_GDT_ENTRY - FIRST_RESERVED_GDT_ENTRY].a = cpu;
order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t));
- idt_tables[cpu] = alloc_xenheap_pages(order, memflags);
+ if ( idt_tables[cpu] == NULL )
+ idt_tables[cpu] = alloc_xenheap_pages(order, memflags);
if ( idt_tables[cpu] == NULL )
goto out;
memcpy(idt_tables[cpu], idt_table, IDT_ENTRIES * sizeof(idt_entry_t));
@@ -1003,16 +1029,16 @@ static int cpu_smpboot_alloc(unsigned int cpu)
(secondary_socket_cpumask = xzalloc(cpumask_t)) == NULL )
goto out;
- if ( !(zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)) &&
- zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)) &&
- alloc_cpumask_var(&per_cpu(scratch_cpumask, cpu))) )
+ if ( !(cond_zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)) &&
+ cond_zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)) &&
+ cond_alloc_cpumask_var(&per_cpu(scratch_cpumask, cpu))) )
goto out;
rc = 0;
out:
if ( rc )
- cpu_smpboot_free(cpu);
+ cpu_smpboot_free(cpu, true);
return rc;
}
@@ -1030,9 +1056,10 @@ static int cpu_smpboot_callback(
break;
case CPU_UP_CANCELED:
case CPU_DEAD:
- cpu_smpboot_free(cpu);
+ cpu_smpboot_free(cpu, !park_offline_cpus);
break;
- default:
+ case CPU_REMOVE:
+ cpu_smpboot_free(cpu, true);
break;
}
diff --git a/xen/arch/x86/spec_ctrl.c b/xen/arch/x86/spec_ctrl.c
index 08e6784c4c..f0c50d6703 100644
--- a/xen/arch/x86/spec_ctrl.c
+++ b/xen/arch/x86/spec_ctrl.c
@@ -19,10 +19,13 @@
#include <xen/errno.h>
#include <xen/init.h>
#include <xen/lib.h>
+#include <xen/warning.h>
#include <asm/microcode.h>
#include <asm/msr.h>
#include <asm/processor.h>
+#include <asm/pv/shim.h>
+#include <asm/setup.h>
#include <asm/spec_ctrl.h>
#include <asm/spec_ctrl_asm.h>
@@ -45,11 +48,16 @@ static int8_t __initdata opt_ibrs = -1;
bool __read_mostly opt_ibpb = true;
bool __read_mostly opt_ssbd = false;
int8_t __read_mostly opt_eager_fpu = -1;
+int8_t __read_mostly opt_l1d_flush = -1;
bool __initdata bsp_delay_spec_ctrl;
uint8_t __read_mostly default_xen_spec_ctrl;
uint8_t __read_mostly default_spec_ctrl_flags;
+paddr_t __read_mostly l1tf_addr_mask, __read_mostly l1tf_safe_maddr;
+static bool __initdata cpu_has_bug_l1tf;
+static unsigned int __initdata l1d_maxphysaddr;
+
static int __init parse_bti(const char *s)
{
const char *ss;
@@ -124,6 +132,17 @@ static int __init parse_spec_ctrl(const char *s)
opt_msr_sc_pv = false;
opt_msr_sc_hvm = false;
+ opt_eager_fpu = 0;
+
+ if ( opt_xpti < 0 )
+ opt_xpti = 0;
+
+ if ( opt_smt < 0 )
+ opt_smt = 1;
+
+ if ( opt_pv_l1tf < 0 )
+ opt_pv_l1tf = 0;
+
disable_common:
opt_rsb_pv = false;
opt_rsb_hvm = false;
@@ -131,7 +150,8 @@ static int __init parse_spec_ctrl(const char *s)
opt_thunk = THUNK_JMP;
opt_ibrs = 0;
opt_ibpb = false;
- opt_eager_fpu = 0;
+ opt_ssbd = false;
+ opt_l1d_flush = 0;
}
else if ( val > 0 )
rc = -EINVAL;
@@ -187,6 +207,8 @@ static int __init parse_spec_ctrl(const char *s)
opt_ssbd = val;
else if ( (val = parse_boolean("eager-fpu", s, ss)) >= 0 )
opt_eager_fpu = val;
+ else if ( (val = parse_boolean("l1d-flush", s, ss)) >= 0 )
+ opt_l1d_flush = val;
else
rc = -EINVAL;
@@ -197,6 +219,55 @@ static int __init parse_spec_ctrl(const char *s)
}
custom_param("spec-ctrl", parse_spec_ctrl);
+int8_t __read_mostly opt_pv_l1tf = -1;
+
+static __init int parse_pv_l1tf(const char *s)
+{
+ const char *ss;
+ int val, rc = 0;
+
+ /* Inhibit the defaults as an explicit choice has been given. */
+ if ( opt_pv_l1tf == -1 )
+ opt_pv_l1tf = 0;
+
+ /* Interpret 'pv-l1tf' alone in its positive boolean form. */
+ if ( *s == '\0' )
+ opt_xpti = OPT_PV_L1TF_DOM0 | OPT_PV_L1TF_DOMU;
+
+ do {
+ ss = strchr(s, ',');
+ if ( !ss )
+ ss = strchr(s, '\0');
+
+ switch ( parse_bool(s, ss) )
+ {
+ case 0:
+ opt_pv_l1tf = 0;
+ break;
+
+ case 1:
+ opt_pv_l1tf = OPT_PV_L1TF_DOM0 | OPT_PV_L1TF_DOMU;
+ break;
+
+ default:
+ if ( (val = parse_boolean("dom0", s, ss)) >= 0 )
+ opt_pv_l1tf = ((opt_pv_l1tf & ~OPT_PV_L1TF_DOM0) |
+ (val ? OPT_PV_L1TF_DOM0 : 0));
+ else if ( (val = parse_boolean("domu", s, ss)) >= 0 )
+ opt_pv_l1tf = ((opt_pv_l1tf & ~OPT_PV_L1TF_DOMU) |
+ (val ? OPT_PV_L1TF_DOMU : 0));
+ else
+ rc = -EINVAL;
+ break;
+ }
+
+ s = ss + 1;
+ } while ( *ss );
+
+ return rc;
+}
+custom_param("pv-l1tf", parse_pv_l1tf);
+
static void __init print_details(enum ind_thunk thunk, uint64_t caps)
{
unsigned int _7d0 = 0, e8b = 0, tmp;
@@ -210,22 +281,31 @@ static void __init print_details(enum ind_thunk thunk, uint64_t caps)
printk("Speculative mitigation facilities:\n");
/* Hardware features which pertain to speculative mitigations. */
- printk(" Hardware features:%s%s%s%s%s%s%s%s\n",
+ printk(" Hardware features:%s%s%s%s%s%s%s%s%s%s\n",
(_7d0 & cpufeat_mask(X86_FEATURE_IBRSB)) ? " IBRS/IBPB" : "",
(_7d0 & cpufeat_mask(X86_FEATURE_STIBP)) ? " STIBP" : "",
+ (_7d0 & cpufeat_mask(X86_FEATURE_L1D_FLUSH)) ? " L1D_FLUSH" : "",
(_7d0 & cpufeat_mask(X86_FEATURE_SSBD)) ? " SSBD" : "",
(e8b & cpufeat_mask(X86_FEATURE_IBPB)) ? " IBPB" : "",
(caps & ARCH_CAPABILITIES_IBRS_ALL) ? " IBRS_ALL" : "",
(caps & ARCH_CAPABILITIES_RDCL_NO) ? " RDCL_NO" : "",
(caps & ARCH_CAPS_RSBA) ? " RSBA" : "",
+ (caps & ARCH_CAPS_SKIP_L1DFL) ? " SKIP_L1DFL": "",
(caps & ARCH_CAPS_SSB_NO) ? " SSB_NO" : "");
- /* Compiled-in support which pertains to BTI mitigations. */
- if ( IS_ENABLED(CONFIG_INDIRECT_THUNK) )
- printk(" Compiled-in support: INDIRECT_THUNK\n");
+ /* Compiled-in support which pertains to mitigations. */
+ if ( IS_ENABLED(CONFIG_INDIRECT_THUNK) || IS_ENABLED(CONFIG_SHADOW_PAGING) )
+ printk(" Compiled-in support:"
+#ifdef CONFIG_INDIRECT_THUNK
+ " INDIRECT_THUNK"
+#endif
+#ifdef CONFIG_SHADOW_PAGING
+ " SHADOW_PAGING"
+#endif
+ "\n");
/* Settings for Xen's protection, irrespective of guests. */
- printk(" Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s, Other:%s\n",
+ printk(" Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s, Other:%s%s\n",
thunk == THUNK_NONE ? "N/A" :
thunk == THUNK_RETPOLINE ? "RETPOLINE" :
thunk == THUNK_LFENCE ? "LFENCE" :
@@ -234,7 +314,15 @@ static void __init print_details(enum ind_thunk thunk, uint64_t caps)
(default_xen_spec_ctrl & SPEC_CTRL_IBRS) ? "IBRS+" : "IBRS-",
!boot_cpu_has(X86_FEATURE_SSBD) ? "" :
(default_xen_spec_ctrl & SPEC_CTRL_SSBD) ? " SSBD+" : " SSBD-",
- opt_ibpb ? " IBPB" : "");
+ opt_ibpb ? " IBPB" : "",
+ opt_l1d_flush ? " L1D_FLUSH" : "");
+
+ /* L1TF diagnostics, printed if vulnerable or PV shadowing is in use. */
+ if ( cpu_has_bug_l1tf || opt_pv_l1tf )
+ printk(" L1TF: believed%s vulnerable, maxphysaddr L1D %u, CPUID %u"
+ ", Safe address %"PRIx64"\n",
+ cpu_has_bug_l1tf ? "" : " not",
+ l1d_maxphysaddr, paddr_bits, l1tf_safe_maddr);
/*
* Alternatives blocks for protecting against and/or virtualising
@@ -257,6 +345,10 @@ static void __init print_details(enum ind_thunk thunk, uint64_t caps)
printk(" XPTI (64-bit PV only): Dom0 %s, DomU %s\n",
opt_xpti & OPT_XPTI_DOM0 ? "enabled" : "disabled",
opt_xpti & OPT_XPTI_DOMU ? "enabled" : "disabled");
+
+ printk(" PV L1TF shadowing: Dom0 %s, DomU %s\n",
+ opt_pv_l1tf & OPT_PV_L1TF_DOM0 ? "enabled" : "disabled",
+ opt_pv_l1tf & OPT_PV_L1TF_DOMU ? "enabled" : "disabled");
}
/* Calculate whether Retpoline is known-safe on this CPU. */
@@ -418,20 +510,159 @@ static bool __init should_use_eager_fpu(void)
}
}
-#define OPT_XPTI_DEFAULT 0xff
-uint8_t __read_mostly opt_xpti = OPT_XPTI_DEFAULT;
-
-static __init void xpti_init_default(bool force)
+/* Calculate whether this CPU is vulnerable to L1TF. */
+static __init void l1tf_calculations(uint64_t caps)
{
- uint64_t caps = 0;
+ bool hit_default = false;
+
+ l1d_maxphysaddr = paddr_bits;
+
+ /* L1TF is only known to affect Intel Family 6 processors at this time. */
+ if ( boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
+ boot_cpu_data.x86 == 6 )
+ {
+ switch ( boot_cpu_data.x86_model )
+ {
+ /*
+ * Core processors since at least Penryn are vulnerable.
+ */
+ case 0x17: /* Penryn */
+ case 0x1d: /* Dunnington */
+ cpu_has_bug_l1tf = true;
+ break;
+
+ case 0x1f: /* Auburndale / Havendale */
+ case 0x1e: /* Nehalem */
+ case 0x1a: /* Nehalem EP */
+ case 0x2e: /* Nehalem EX */
+ case 0x25: /* Westmere */
+ case 0x2c: /* Westmere EP */
+ case 0x2f: /* Westmere EX */
+ cpu_has_bug_l1tf = true;
+ l1d_maxphysaddr = 44;
+ break;
+
+ case 0x2a: /* SandyBridge */
+ case 0x2d: /* SandyBridge EP/EX */
+ case 0x3a: /* IvyBridge */
+ case 0x3e: /* IvyBridge EP/EX */
+ case 0x3c: /* Haswell */
+ case 0x3f: /* Haswell EX/EP */
+ case 0x45: /* Haswell D */
+ case 0x46: /* Haswell H */
+ case 0x3d: /* Broadwell */
+ case 0x47: /* Broadwell H */
+ case 0x4f: /* Broadwell EP/EX */
+ case 0x56: /* Broadwell D */
+ case 0x4e: /* Skylake M */
+ case 0x55: /* Skylake X */
+ case 0x5e: /* Skylake D */
+ case 0x66: /* Cannonlake */
+ case 0x67: /* Cannonlake? */
+ case 0x8e: /* Kabylake M */
+ case 0x9e: /* Kabylake D */
+ cpu_has_bug_l1tf = true;
+ l1d_maxphysaddr = 46;
+ break;
+
+ /*
+ * Atom processors are not vulnerable.
+ */
+ case 0x1c: /* Pineview */
+ case 0x26: /* Lincroft */
+ case 0x27: /* Penwell */
+ case 0x35: /* Cloverview */
+ case 0x36: /* Cedarview */
+ case 0x37: /* Baytrail / Valleyview (Silvermont) */
+ case 0x4d: /* Avaton / Rangely (Silvermont) */
+ case 0x4c: /* Cherrytrail / Brasswell */
+ case 0x4a: /* Merrifield */
+ case 0x5a: /* Moorefield */
+ case 0x5c: /* Goldmont */
+ case 0x5f: /* Denverton */
+ case 0x7a: /* Gemini Lake */
+ break;
+
+ /*
+ * Knights processors are not vulnerable.
+ */
+ case 0x57: /* Knights Landing */
+ case 0x85: /* Knights Mill */
+ break;
+
+ default:
+ /* Defer printk() until we've accounted for RDCL_NO. */
+ hit_default = true;
+ cpu_has_bug_l1tf = true;
+ break;
+ }
+ }
+
+ /* Any processor advertising RDCL_NO should be not vulnerable to L1TF. */
+ if ( caps & ARCH_CAPABILITIES_RDCL_NO )
+ cpu_has_bug_l1tf = false;
+
+ if ( cpu_has_bug_l1tf && hit_default )
+ printk("Unrecognised CPU model %#x - assuming vulnerable to L1TF\n",
+ boot_cpu_data.x86_model);
+
+ /*
+ * L1TF safe address heuristics. These apply to the real hardware we are
+ * running on, and are best-effort-only if Xen is virtualised.
+ *
+ * The address mask which the L1D cache uses, which might be wider than
+ * the CPUID-reported maxphysaddr.
+ */
+ l1tf_addr_mask = ((1ul << l1d_maxphysaddr) - 1) & PAGE_MASK;
+
+ /*
+ * To be safe, l1tf_safe_maddr must be above the highest cacheable entity
+ * in system physical address space. However, to preserve space for
+ * paged-out metadata, it should be as low as possible above the highest
+ * cacheable address, so as to require fewer high-order bits being set.
+ *
+ * These heuristics are based on some guesswork to improve the likelihood
+ * of safety in the common case, including Linux's L1TF mitigation of
+ * inverting all address bits in a non-present PTE.
+ *
+ * - If L1D is wider than CPUID (Nehalem and later mobile/desktop/low end
+ * server), setting any address bit beyond CPUID maxphysaddr guarantees
+ * to make the PTE safe. This case doesn't require all the high-order
+ * bits being set, and doesn't require any other source of information
+ * for safety.
+ *
+ * - If L1D is the same as CPUID (Pre-Nehalem, or high end server), we
+ * must sacrifice high order bits from the real address space for
+ * safety. Therefore, make a blind guess that there is nothing
+ * cacheable in the top quarter of physical address space.
+ *
+ * It is exceedingly unlikely for machines to be populated with this
+ * much RAM (likely 512G on pre-Nehalem, 16T on Nehalem/Westmere, 64T on
+ * Sandybridge and later) due to the sheer volume of DIMMs this would
+ * actually take.
+ *
+ * However, it is possible to find machines this large, so the "top
+ * quarter" guess is supplemented to push the limit higher if references
+ * to cacheable mappings (E820/SRAT/EFI/etc) are found above the top
+ * quarter boundary.
+ *
+ * Finally, this top quarter guess gives us a good chance of being safe
+ * when running virtualised (and the CPUID maxphysaddr hasn't been
+ * levelled for heterogeneous migration safety), where the safety
+ * consideration is still in terms of host details, but all E820/etc
+ * information is in terms of guest physical layout.
+ */
+ l1tf_safe_maddr = max(l1tf_safe_maddr, ((l1d_maxphysaddr > paddr_bits)
+ ? (1ul << paddr_bits)
+ : (3ul << (paddr_bits - 2))));
+}
- if ( !force && (opt_xpti != OPT_XPTI_DEFAULT) )
- return;
+int8_t __read_mostly opt_xpti = -1;
+static __init void xpti_init_default(uint64_t caps)
+{
if ( boot_cpu_data.x86_vendor == X86_VENDOR_AMD )
caps = ARCH_CAPABILITIES_RDCL_NO;
- else if ( boot_cpu_has(X86_FEATURE_ARCH_CAPS) )
- rdmsrl(MSR_ARCH_CAPABILITIES, caps);
if ( caps & ARCH_CAPABILITIES_RDCL_NO )
opt_xpti = 0;
@@ -444,7 +675,13 @@ static __init int parse_xpti(const char *s)
const char *ss;
int val, rc = 0;
- xpti_init_default(false);
+ /* Inhibit the defaults as an explicit choice has been given. */
+ if ( opt_xpti == -1 )
+ opt_xpti = 0;
+
+ /* Interpret 'xpti' alone in its positive boolean form. */
+ if ( *s == '\0' )
+ opt_xpti = OPT_XPTI_DOM0 | OPT_XPTI_DOMU;
do {
ss = strchr(s, ',');
@@ -463,7 +700,7 @@ static __init int parse_xpti(const char *s)
default:
if ( !strcmp(s, "default") )
- xpti_init_default(true);
+ opt_xpti = -1;
else if ( (val = parse_boolean("dom0", s, ss)) >= 0 )
opt_xpti = (opt_xpti & ~OPT_XPTI_DOM0) |
(val ? OPT_XPTI_DOM0 : 0);
@@ -625,12 +862,58 @@ void __init init_speculation_mitigations(void)
if ( default_xen_spec_ctrl )
setup_force_cpu_cap(X86_FEATURE_SC_MSR_IDLE);
- xpti_init_default(false);
+ if ( opt_xpti == -1 )
+ xpti_init_default(caps);
+
if ( opt_xpti == 0 )
setup_force_cpu_cap(X86_FEATURE_NO_XPTI);
else
setup_clear_cpu_cap(X86_FEATURE_NO_XPTI);
+ l1tf_calculations(caps);
+
+ /*
+ * By default, enable PV domU L1TF mitigations on all L1TF-vulnerable
+ * hardware, except when running in shim mode.
+ *
+ * In shim mode, SHADOW is expected to be compiled out, and a malicious
+ * guest kernel can only attack the shim Xen, not the host Xen.
+ */
+ if ( opt_pv_l1tf == -1 )
+ {
+ if ( pv_shim || !cpu_has_bug_l1tf )
+ opt_pv_l1tf = 0;
+ else
+ opt_pv_l1tf = OPT_PV_L1TF_DOMU;
+ }
+
+ /*
+ * By default, enable L1D_FLUSH on L1TF-vulnerable hardware, unless
+ * instructed to skip the flush on vmentry by our outer hypervisor.
+ */
+ if ( !boot_cpu_has(X86_FEATURE_L1D_FLUSH) )
+ opt_l1d_flush = 0;
+ else if ( opt_l1d_flush == -1 )
+ opt_l1d_flush = cpu_has_bug_l1tf && !(caps & ARCH_CAPS_SKIP_L1DFL);
+
+ /*
+ * We do not disable HT by default on affected hardware.
+ *
+ * Firstly, if the user intends to use exclusively PV, or HVM shadow
+ * guests, HT isn't a concern and should remain fully enabled. Secondly,
+ * safety for HVM HAP guests can be arranged by the toolstack with core
+ * parking, pinning or cpupool configurations, including mixed setups.
+ *
+ * However, if we are on affected hardware, with HT enabled, and the user
+ * hasn't explicitly chosen whether to use HT or not, nag them to do so.
+ */
+ if ( opt_smt == -1 && cpu_has_bug_l1tf && !pv_shim &&
+ boot_cpu_data.x86_num_siblings > 1 )
+ warning_add(
+ "Booted on L1TF-vulnerable hardware with SMT/Hyperthreading\n"
+ "enabled. Please assess your configuration and choose an\n"
+ "explicit 'smt=<bool>' setting. See XSA-273.\n");
+
print_details(thunk, caps);
/*
diff --git a/xen/arch/x86/srat.c b/xen/arch/x86/srat.c
index 166eb44fe2..2d70b45909 100644
--- a/xen/arch/x86/srat.c
+++ b/xen/arch/x86/srat.c
@@ -20,6 +20,7 @@
#include <xen/pfn.h>
#include <asm/e820.h>
#include <asm/page.h>
+#include <asm/spec_ctrl.h>
static struct acpi_table_slit *__read_mostly acpi_slit;
@@ -284,6 +285,11 @@ acpi_numa_memory_affinity_init(const struct acpi_srat_mem_affinity *ma)
if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
return;
+ start = ma->base_address;
+ end = start + ma->length;
+ /* Supplement the heuristics in l1tf_calculations(). */
+ l1tf_safe_maddr = max(l1tf_safe_maddr, ROUNDUP(end, PAGE_SIZE));
+
if (num_node_memblks >= NR_NODE_MEMBLKS)
{
dprintk(XENLOG_WARNING,
@@ -292,8 +298,6 @@ acpi_numa_memory_affinity_init(const struct acpi_srat_mem_affinity *ma)
return;
}
- start = ma->base_address;
- end = start + ma->length;
pxm = ma->proximity_domain;
if (srat_rev < 2)
pxm &= 0xff;
diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c
index 4d372db12b..e704ed7f1c 100644
--- a/xen/arch/x86/sysctl.c
+++ b/xen/arch/x86/sysctl.c
@@ -23,6 +23,7 @@
#include <asm/hvm/hvm.h>
#include <asm/hvm/support.h>
#include <asm/processor.h>
+#include <asm/setup.h>
#include <asm/smp.h>
#include <asm/numa.h>
#include <xen/nodemask.h>
@@ -48,14 +49,27 @@ static void l3_cache_get(void *arg)
long cpu_up_helper(void *data)
{
- int cpu = (unsigned long)data;
+ unsigned int cpu = (unsigned long)data;
int ret = cpu_up(cpu);
+
if ( ret == -EBUSY )
{
/* On EBUSY, flush RCU work and have one more go. */
rcu_barrier();
ret = cpu_up(cpu);
}
+
+ if ( !ret && !opt_smt &&
+ cpu_data[cpu].compute_unit_id == INVALID_CUID &&
+ cpumask_weight(per_cpu(cpu_sibling_mask, cpu)) > 1 )
+ {
+ ret = cpu_down_helper(data);
+ if ( ret )
+ printk("Could not re-offline CPU%u (%d)\n", cpu, ret);
+ else
+ ret = -EPERM;
+ }
+
return ret;
}
diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index 9f045a2045..789d7ff8cd 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -96,8 +96,6 @@ string_param("nmi", opt_nmi);
DEFINE_PER_CPU(uint64_t, efer);
static DEFINE_PER_CPU(unsigned long, last_extable_addr);
-DEFINE_PER_CPU_READ_MOSTLY(u32, ler_msr);
-
DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, gdt_table);
DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, compat_gdt_table);
@@ -117,6 +115,9 @@ integer_param("debug_stack_lines", debug_stack_lines);
static bool opt_ler;
boolean_param("ler", opt_ler);
+/* LastExceptionFromIP on this hardware. Zero if LER is not in use. */
+unsigned int __read_mostly ler_msr;
+
#define stack_words_per_line 4
#define ESP_BEFORE_EXCEPTION(regs) ((unsigned long *)regs->rsp)
@@ -1778,17 +1779,6 @@ void do_device_not_available(struct cpu_user_regs *regs)
return;
}
-static void ler_enable(void)
-{
- u64 debugctl;
-
- if ( !this_cpu(ler_msr) )
- return;
-
- rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
- wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl | IA32_DEBUGCTLMSR_LBR);
-}
-
void do_debug(struct cpu_user_regs *regs)
{
unsigned long dr6;
@@ -1821,6 +1811,10 @@ void do_debug(struct cpu_user_regs *regs)
*/
write_debugreg(6, X86_DR6_DEFAULT);
+ /* #DB automatically disabled LBR. Reinstate it if debugging Xen. */
+ if ( cpu_has_xen_lbr )
+ wrmsrl(MSR_IA32_DEBUGCTLMSR, IA32_DEBUGCTLMSR_LBR);
+
if ( !guest_mode(regs) )
{
/*
@@ -1838,7 +1832,7 @@ void do_debug(struct cpu_user_regs *regs)
{
if ( regs->rip == (unsigned long)sysenter_eflags_saved )
regs->eflags &= ~X86_EFLAGS_TF;
- goto out;
+ return;
}
if ( !debugger_trap_fatal(TRAP_debug, regs) )
{
@@ -1895,20 +1889,14 @@ void do_debug(struct cpu_user_regs *regs)
regs->cs, _p(regs->rip), _p(regs->rip),
regs->ss, _p(regs->rsp), dr6);
- goto out;
+ return;
}
/* Save debug status register where guest OS can peek at it */
v->arch.debugreg[6] |= (dr6 & ~X86_DR6_DEFAULT);
v->arch.debugreg[6] &= (dr6 | ~X86_DR6_DEFAULT);
- ler_enable();
pv_inject_hw_exception(TRAP_debug, X86_EVENT_NO_EC);
- return;
-
- out:
- ler_enable();
- return;
}
static void __init noinline __set_intr_gate(unsigned int n,
@@ -1952,38 +1940,46 @@ void load_TR(void)
: "=m" (old_gdt) : "rm" (TSS_ENTRY << 3), "m" (tss_gdt) : "memory" );
}
-void percpu_traps_init(void)
+static unsigned int calc_ler_msr(void)
{
- subarch_percpu_traps_init();
-
- if ( !opt_ler )
- return;
-
switch ( boot_cpu_data.x86_vendor )
{
case X86_VENDOR_INTEL:
switch ( boot_cpu_data.x86 )
{
case 6:
- this_cpu(ler_msr) = MSR_IA32_LASTINTFROMIP;
- break;
+ return MSR_IA32_LASTINTFROMIP;
+
case 15:
- this_cpu(ler_msr) = MSR_P4_LER_FROM_LIP;
- break;
+ return MSR_P4_LER_FROM_LIP;
}
break;
+
case X86_VENDOR_AMD:
switch ( boot_cpu_data.x86 )
{
case 6:
case 0xf ... 0x17:
- this_cpu(ler_msr) = MSR_IA32_LASTINTFROMIP;
- break;
+ return MSR_IA32_LASTINTFROMIP;
}
break;
}
- ler_enable();
+ return 0;
+}
+
+void percpu_traps_init(void)
+{
+ subarch_percpu_traps_init();
+
+ if ( !opt_ler )
+ return;
+
+ if ( !ler_msr && (ler_msr = calc_ler_msr()) )
+ setup_force_cpu_cap(X86_FEATURE_XEN_LBR);
+
+ if ( cpu_has_xen_lbr )
+ wrmsrl(MSR_IA32_DEBUGCTLMSR, IA32_DEBUGCTLMSR_LBR);
}
void __init init_idt_traps(void)
diff --git a/xen/arch/x86/x86_64/compat/mm.c b/xen/arch/x86/x86_64/compat/mm.c
index c2aa6f2fdb..02bc75b91e 100644
--- a/xen/arch/x86/x86_64/compat/mm.c
+++ b/xen/arch/x86/x86_64/compat/mm.c
@@ -163,19 +163,6 @@ int compat_arch_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)
return rc;
}
-int compat_update_va_mapping(unsigned int va, u32 lo, u32 hi,
- unsigned int flags)
-{
- return do_update_va_mapping(va, lo | ((u64)hi << 32), flags);
-}
-
-int compat_update_va_mapping_otherdomain(unsigned long va, u32 lo, u32 hi,
- unsigned long flags,
- domid_t domid)
-{
- return do_update_va_mapping_otherdomain(va, lo | ((u64)hi << 32), flags, domid);
-}
-
DEFINE_XEN_GUEST_HANDLE(mmuext_op_compat_t);
int compat_mmuext_op(XEN_GUEST_HANDLE_PARAM(void) arg,
diff --git a/xen/arch/x86/x86_64/traps.c b/xen/arch/x86/x86_64/traps.c
index f7f6928d70..b0401850ef 100644
--- a/xen/arch/x86/x86_64/traps.c
+++ b/xen/arch/x86/x86_64/traps.c
@@ -144,11 +144,12 @@ void show_registers(const struct cpu_user_regs *regs)
printk("CPU: %d\n", smp_processor_id());
_show_registers(&fault_regs, fault_crs, context, v);
- if ( this_cpu(ler_msr) && !guest_mode(regs) )
+ if ( ler_msr && !guest_mode(regs) )
{
u64 from, to;
- rdmsrl(this_cpu(ler_msr), from);
- rdmsrl(this_cpu(ler_msr) + 1, to);
+
+ rdmsrl(ler_msr, from);
+ rdmsrl(ler_msr + 1, to);
printk("ler: %016lx -> %016lx\n", from, to);
}
}
diff --git a/xen/arch/x86/xstate.c b/xen/arch/x86/xstate.c
index b4aea4b50a..15edd5df96 100644
--- a/xen/arch/x86/xstate.c
+++ b/xen/arch/x86/xstate.c
@@ -670,12 +670,17 @@ static bool valid_xcr0(u64 xcr0)
return !(xcr0 & X86_XCR0_BNDREGS) == !(xcr0 & X86_XCR0_BNDCSR);
}
-int validate_xstate(u64 xcr0, u64 xcr0_accum, const struct xsave_hdr *hdr)
+int validate_xstate(const struct domain *d, uint64_t xcr0, uint64_t xcr0_accum,
+ const struct xsave_hdr *hdr)
{
+ const struct cpuid_policy *cp = d->arch.cpuid;
+ uint64_t xcr0_max =
+ ((uint64_t)cp->xstate.xcr0_high << 32) | cp->xstate.xcr0_low;
unsigned int i;
if ( (hdr->xstate_bv & ~xcr0_accum) ||
(xcr0 & ~xcr0_accum) ||
+ (xcr0_accum & ~xcr0_max) ||
!valid_xcr0(xcr0) ||
!valid_xcr0(xcr0_accum) )
return -EINVAL;
@@ -694,20 +699,40 @@ int validate_xstate(u64 xcr0, u64 xcr0_accum, const struct xsave_hdr *hdr)
int handle_xsetbv(u32 index, u64 new_bv)
{
struct vcpu *curr = current;
+ const struct cpuid_policy *cp = curr->domain->arch.cpuid;
+ uint64_t xcr0_max =
+ ((uint64_t)cp->xstate.xcr0_high << 32) | cp->xstate.xcr0_low;
u64 mask;
if ( index != XCR_XFEATURE_ENABLED_MASK )
return -EOPNOTSUPP;
- if ( (new_bv & ~xfeature_mask) || !valid_xcr0(new_bv) )
+ /*
+ * The CPUID logic shouldn't be able to hand out an XCR0 exceeding Xen's
+ * maximum features, but keep the check for robustness.
+ */
+ if ( unlikely(xcr0_max & ~xfeature_mask) )
+ {
+ gprintk(XENLOG_ERR,
+ "xcr0_max %016" PRIx64 " exceeds hardware max %016" PRIx64 "\n",
+ xcr0_max, xfeature_mask);
+ domain_crash(curr->domain);
+
+ return -EINVAL;
+ }
+
+ if ( (new_bv & ~xcr0_max) || !valid_xcr0(new_bv) )
return -EINVAL;
- /* XCR0.PKRU is disabled on PV mode. */
- if ( is_pv_vcpu(curr) && (new_bv & X86_XCR0_PKRU) )
- return -EOPNOTSUPP;
+ /* By this point, new_bv really should be accepted by hardware. */
+ if ( unlikely(!set_xcr0(new_bv)) )
+ {
+ gprintk(XENLOG_ERR, "new_bv %016" PRIx64 " rejected by hardware\n",
+ new_bv);
+ domain_crash(curr->domain);
- if ( !set_xcr0(new_bv) )
return -EFAULT;
+ }
mask = new_bv & ~curr->arch.xcr0_accum;
curr->arch.xcr0 = new_bv;
diff --git a/xen/common/cpu.c b/xen/common/cpu.c
index 6350f150bd..653a56b840 100644
--- a/xen/common/cpu.c
+++ b/xen/common/cpu.c
@@ -67,12 +67,17 @@ void __init register_cpu_notifier(struct notifier_block *nb)
spin_unlock(&cpu_add_remove_lock);
}
-static int take_cpu_down(void *unused)
+static void _take_cpu_down(void *unused)
{
void *hcpu = (void *)(long)smp_processor_id();
int notifier_rc = notifier_call_chain(&cpu_chain, CPU_DYING, hcpu, NULL);
BUG_ON(notifier_rc != NOTIFY_DONE);
__cpu_disable();
+}
+
+static int take_cpu_down(void *arg)
+{
+ _take_cpu_down(arg);
return 0;
}
@@ -98,7 +103,9 @@ int cpu_down(unsigned int cpu)
goto fail;
}
- if ( (err = stop_machine_run(take_cpu_down, NULL, cpu)) < 0 )
+ if ( unlikely(system_state < SYS_STATE_active) )
+ on_selected_cpus(cpumask_of(cpu), _take_cpu_down, NULL, true);
+ else if ( (err = stop_machine_run(take_cpu_down, NULL, cpu)) < 0 )
goto fail;
__cpu_die(cpu);
diff --git a/xen/common/cpupool.c b/xen/common/cpupool.c
index 999839444e..1e8edcbd57 100644
--- a/xen/common/cpupool.c
+++ b/xen/common/cpupool.c
@@ -490,7 +490,7 @@ static int cpupool_cpu_add(unsigned int cpu)
cpumask_clear_cpu(cpu, &cpupool_locked_cpus);
cpumask_set_cpu(cpu, &cpupool_free_cpus);
- if ( system_state == SYS_STATE_resume )
+ if ( system_state == SYS_STATE_suspend || system_state == SYS_STATE_resume )
{
struct cpupool **c;
@@ -522,6 +522,7 @@ static int cpupool_cpu_add(unsigned int cpu)
* (or unplugging would have failed) and that is the default behavior
* anyway.
*/
+ per_cpu(cpupool, cpu) = NULL;
ret = cpupool_assign_cpu_locked(cpupool0, cpu);
}
out:
diff --git a/xen/common/efi/boot.c b/xen/common/efi/boot.c
index 64d12685d3..6be0b3986f 100644
--- a/xen/common/efi/boot.c
+++ b/xen/common/efi/boot.c
@@ -1304,6 +1304,8 @@ efi_start(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE *SystemTable)
#ifndef CONFIG_ARM /* TODO - runtime service support */
+#include <asm/spec_ctrl.h>
+
static bool __initdata efi_map_uc;
static int __init parse_efi_param(const char *s)
@@ -1419,6 +1421,16 @@ void __init efi_init_memory(void)
desc->PhysicalStart, desc->PhysicalStart + len - 1,
desc->Type, desc->Attribute);
+ if ( (desc->Attribute & (EFI_MEMORY_WB | EFI_MEMORY_WT)) ||
+ (efi_bs_revision >= EFI_REVISION(2, 5) &&
+ (desc->Attribute & EFI_MEMORY_WP)) )
+ {
+ /* Supplement the heuristics in l1tf_calculations(). */
+ l1tf_safe_maddr =
+ max(l1tf_safe_maddr,
+ ROUNDUP(desc->PhysicalStart + len, PAGE_SIZE));
+ }
+
if ( !efi_enabled(EFI_RS) ||
(!(desc->Attribute & EFI_MEMORY_RUNTIME) &&
(!map_bs ||
diff --git a/xen/common/grant_table.c b/xen/common/grant_table.c
index c757b7f6f5..231ecf509a 100644
--- a/xen/common/grant_table.c
+++ b/xen/common/grant_table.c
@@ -97,7 +97,11 @@ static unsigned int __read_mostly max_maptrack_frames =
DEFAULT_MAX_MAPTRACK_FRAMES;
integer_runtime_param("gnttab_max_maptrack_frames", max_maptrack_frames);
-static unsigned int __read_mostly opt_gnttab_max_version = 2;
+#ifndef GNTTAB_MAX_VERSION
+#define GNTTAB_MAX_VERSION 2
+#endif
+
+static unsigned int __read_mostly opt_gnttab_max_version = GNTTAB_MAX_VERSION;
static bool __read_mostly opt_transitive_grants = true;
static int __init parse_gnttab(const char *s)
diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c
index 20ee1e4897..02aeed7c47 100644
--- a/xen/common/page_alloc.c
+++ b/xen/common/page_alloc.c
@@ -1426,7 +1426,7 @@ static void free_heap_pages(
page_list_del(predecessor, &heap(node, zone, order));
- /* Keep predecessor's first_dirty if it is already set. */
+ /* Update predecessor's first_dirty if necessary. */
if ( predecessor->u.free.first_dirty == INVALID_DIRTY_IDX &&
pg->u.free.first_dirty != INVALID_DIRTY_IDX )
predecessor->u.free.first_dirty = (1U << order) +
@@ -1447,6 +1447,12 @@ static void free_heap_pages(
check_and_stop_scrub(successor);
+ /* Update pg's first_dirty if necessary. */
+ if ( pg->u.free.first_dirty == INVALID_DIRTY_IDX &&
+ successor->u.free.first_dirty != INVALID_DIRTY_IDX )
+ pg->u.free.first_dirty = (1U << order) +
+ successor->u.free.first_dirty;
+
page_list_del(successor, &heap(node, zone, order));
}
diff --git a/xen/common/tasklet.c b/xen/common/tasklet.c
index 0f0a6f8365..d4fea3151c 100644
--- a/xen/common/tasklet.c
+++ b/xen/common/tasklet.c
@@ -156,6 +156,10 @@ void tasklet_kill(struct tasklet *t)
spin_lock_irqsave(&tasklet_lock, flags);
+ /* Cope with uninitialised tasklets. */
+ if ( list_head_is_null(&t->list) )
+ goto unlock;
+
if ( !list_empty(&t->list) )
{
BUG_ON(t->is_dead || t->is_running || (t->scheduled_on < 0));
@@ -172,6 +176,7 @@ void tasklet_kill(struct tasklet *t)
spin_lock_irqsave(&tasklet_lock, flags);
}
+ unlock:
spin_unlock_irqrestore(&tasklet_lock, flags);
}
diff --git a/xen/include/asm-arm/arm32/system.h b/xen/include/asm-arm/arm32/system.h
index c617b40438..ab57abfbc5 100644
--- a/xen/include/asm-arm/arm32/system.h
+++ b/xen/include/asm-arm/arm32/system.h
@@ -48,6 +48,24 @@ static inline int local_fiq_is_enabled(void)
return !(flags & PSR_FIQ_MASK);
}
+#define CSDB ".inst 0xe320f014"
+
+static inline unsigned long array_index_mask_nospec(unsigned long idx,
+ unsigned long sz)
+{
+ unsigned long mask;
+
+ asm volatile( "cmp %1, %2\n"
+ "sbc %0, %1, %1\n"
+ CSDB
+ : "=r" (mask)
+ : "r" (idx), "Ir" (sz)
+ : "cc" );
+
+ return mask;
+}
+#define array_index_mask_nospec array_index_mask_nospec
+
#endif
/*
* Local variables:
diff --git a/xen/include/asm-arm/arm64/system.h b/xen/include/asm-arm/arm64/system.h
index 2e2ee212a1..2e36573ac6 100644
--- a/xen/include/asm-arm/arm64/system.h
+++ b/xen/include/asm-arm/arm64/system.h
@@ -58,6 +58,28 @@ static inline int local_fiq_is_enabled(void)
return !(flags & PSR_FIQ_MASK);
}
+#define csdb() asm volatile ( "hint #20" : : : "memory" )
+
+/*
+ * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz
+ * and 0 otherwise.
+ */
+static inline unsigned long array_index_mask_nospec(unsigned long idx,
+ unsigned long sz)
+{
+ unsigned long mask;
+
+ asm volatile ( "cmp %1, %2\n"
+ "sbc %0, xzr, xzr\n"
+ : "=r" (mask)
+ : "r" (idx), "Ir" (sz)
+ : "cc" );
+ csdb();
+
+ return mask;
+}
+#define array_index_mask_nospec array_index_mask_nospec
+
#endif
/*
* Local variables:
diff --git a/xen/include/asm-arm/grant_table.h b/xen/include/asm-arm/grant_table.h
index e52936c79f..24958e4670 100644
--- a/xen/include/asm-arm/grant_table.h
+++ b/xen/include/asm-arm/grant_table.h
@@ -7,6 +7,7 @@
#include <xen/sched.h>
#define INITIAL_NR_GRANT_FRAMES 1U
+#define GNTTAB_MAX_VERSION 1
struct grant_table_arch {
gfn_t *shared_gfn;
diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h
index 2cf8f7ea2a..b237da165c 100644
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -113,6 +113,7 @@
#define cpu_has_aperfmperf boot_cpu_has(X86_FEATURE_APERFMPERF)
#define cpu_has_lfence_dispatch boot_cpu_has(X86_FEATURE_LFENCE_DISPATCH)
#define cpu_has_no_xpti boot_cpu_has(X86_FEATURE_NO_XPTI)
+#define cpu_has_xen_lbr boot_cpu_has(X86_FEATURE_XEN_LBR)
enum _cache_type {
CACHE_TYPE_NULL = 0,
diff --git a/xen/include/asm-x86/cpufeatures.h b/xen/include/asm-x86/cpufeatures.h
index b90aa2d046..8e5cc53dde 100644
--- a/xen/include/asm-x86/cpufeatures.h
+++ b/xen/include/asm-x86/cpufeatures.h
@@ -32,3 +32,4 @@ XEN_CPUFEATURE(SC_RSB_PV, (FSCAPINTS+0)*32+18) /* RSB overwrite needed for
XEN_CPUFEATURE(SC_RSB_HVM, (FSCAPINTS+0)*32+19) /* RSB overwrite needed for HVM */
XEN_CPUFEATURE(NO_XPTI, (FSCAPINTS+0)*32+20) /* XPTI mitigation not in use */
XEN_CPUFEATURE(SC_MSR_IDLE, (FSCAPINTS+0)*32+21) /* (SC_MSR_PV || SC_MSR_HVM) && default_xen_spec_ctrl */
+XEN_CPUFEATURE(XEN_LBR, (FSCAPINTS+0)*32+22) /* Xen uses MSR_DEBUGCTL.LBR */
diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
index e0d413c7de..61e6900465 100644
--- a/xen/include/asm-x86/domain.h
+++ b/xen/include/asm-x86/domain.h
@@ -121,6 +121,11 @@ struct shadow_domain {
/* Has this domain ever used HVMOP_pagetable_dying? */
bool_t pagetable_dying_op;
+
+#ifdef CONFIG_PV
+ /* PV L1 Terminal Fault mitigation. */
+ struct tasklet pv_l1tf_tasklet;
+#endif /* CONFIG_PV */
#endif
};
@@ -257,6 +262,8 @@ struct pv_domain
bool xpti;
/* Use PCID feature? */
bool pcid;
+ /* Mitigate L1TF with shadow/crashing? */
+ bool check_l1tf;
/* map_domain_page() mapping cache. */
struct mapcache_domain mapcache;
diff --git a/xen/include/asm-x86/hvm/vmx/vmcs.h b/xen/include/asm-x86/hvm/vmx/vmcs.h
index 06c3179cec..57e5098b99 100644
--- a/xen/include/asm-x86/hvm/vmx/vmcs.h
+++ b/xen/include/asm-x86/hvm/vmx/vmcs.h
@@ -130,10 +130,18 @@ struct arch_vmx_struct {
uint64_t sfmask;
struct vmx_msr_bitmap *msr_bitmap;
- unsigned int msr_count;
+
+ /*
+ * Most accesses to the MSR host/guest load/save lists are in current
+ * context. However, the data can be modified by toolstack/migration
+ * actions. Remote access is only permitted for paused vcpus, and is
+ * protected under the domctl lock.
+ */
struct vmx_msr_entry *msr_area;
- unsigned int host_msr_count;
struct vmx_msr_entry *host_msr_area;
+ unsigned int msr_load_count;
+ unsigned int msr_save_count;
+ unsigned int host_msr_count;
unsigned long eoi_exitmap_changed;
DECLARE_BITMAP(eoi_exit_bitmap, NR_VECTORS);
@@ -149,7 +157,7 @@ struct arch_vmx_struct {
/* Are we emulating rather than VMENTERing? */
uint8_t vmx_emulate;
- uint8_t lbr_fixup_enabled;
+ uint8_t lbr_flags;
/* Bitmask of segments that we can't safely use in virtual 8086 mode */
uint16_t vm86_segment_mask;
@@ -514,9 +522,6 @@ enum vmcs_field {
#define VMCS_VPID_WIDTH 16
-#define VMX_GUEST_MSR 0
-#define VMX_HOST_MSR 1
-
/* VM Instruction error numbers */
enum vmx_insn_errno
{
@@ -534,6 +539,67 @@ enum vmx_insn_errno
VMX_INSN_FAIL_INVALID = ~0,
};
+/* MSR load/save list infrastructure. */
+enum vmx_msr_list_type {
+ VMX_MSR_HOST, /* MSRs loaded on VMExit. */
+ VMX_MSR_GUEST, /* MSRs saved on VMExit, loaded on VMEntry. */
+ VMX_MSR_GUEST_LOADONLY, /* MSRs loaded on VMEntry only. */
+};
+
+/**
+ * Add an MSR to an MSR list (inserting space for the entry if necessary), and
+ * set the MSRs value.
+ *
+ * It is undefined behaviour to try and insert the same MSR into both the
+ * GUEST and GUEST_LOADONLY list.
+ *
+ * May fail if unable to allocate memory for the list, or the total number of
+ * entries exceeds the memory allocated.
+ */
+int vmx_add_msr(struct vcpu *v, uint32_t msr, uint64_t val,
+ enum vmx_msr_list_type type);
+
+static inline int vmx_add_guest_msr(struct vcpu *v, uint32_t msr, uint64_t val)
+{
+ return vmx_add_msr(v, msr, val, VMX_MSR_GUEST);
+}
+static inline int vmx_add_host_load_msr(struct vcpu *v, uint32_t msr,
+ uint64_t val)
+{
+ return vmx_add_msr(v, msr, val, VMX_MSR_HOST);
+}
+
+struct vmx_msr_entry *vmx_find_msr(const struct vcpu *v, uint32_t msr,
+ enum vmx_msr_list_type type);
+
+static inline int vmx_read_guest_msr(const struct vcpu *v, uint32_t msr,
+ uint64_t *val)
+{
+ const struct vmx_msr_entry *ent = vmx_find_msr(v, msr, VMX_MSR_GUEST);
+
+ if ( !ent )
+ return -ESRCH;
+
+ *val = ent->data;
+
+ return 0;
+}
+
+static inline int vmx_write_guest_msr(struct vcpu *v, uint32_t msr,
+ uint64_t val)
+{
+ struct vmx_msr_entry *ent = vmx_find_msr(v, msr, VMX_MSR_GUEST);
+
+ if ( !ent )
+ return -ESRCH;
+
+ ent->data = val;
+
+ return 0;
+}
+
+
+/* MSR intercept bitmap infrastructure. */
enum vmx_msr_intercept_type {
VMX_MSR_R = 1,
VMX_MSR_W = 2,
@@ -544,10 +610,6 @@ void vmx_clear_msr_intercept(struct vcpu *v, unsigned int msr,
enum vmx_msr_intercept_type type);
void vmx_set_msr_intercept(struct vcpu *v, unsigned int msr,
enum vmx_msr_intercept_type type);
-int vmx_read_guest_msr(u32 msr, u64 *val);
-int vmx_write_guest_msr(u32 msr, u64 val);
-struct vmx_msr_entry *vmx_find_msr(u32 msr, int type);
-int vmx_add_msr(u32 msr, int type);
void vmx_vmcs_switch(paddr_t from, paddr_t to);
void vmx_set_eoi_exit_bitmap(struct vcpu *v, u8 vector);
void vmx_clear_eoi_exit_bitmap(struct vcpu *v, u8 vector);
@@ -562,15 +624,6 @@ void virtual_vmcs_vmwrite(const struct vcpu *, u32 encoding, u64 val);
enum vmx_insn_errno virtual_vmcs_vmwrite_safe(const struct vcpu *v,
u32 vmcs_encoding, u64 val);
-static inline int vmx_add_guest_msr(u32 msr)
-{
- return vmx_add_msr(msr, VMX_GUEST_MSR);
-}
-static inline int vmx_add_host_load_msr(u32 msr)
-{
- return vmx_add_msr(msr, VMX_HOST_MSR);
-}
-
DECLARE_PER_CPU(bool_t, vmxon);
bool_t vmx_vcpu_pml_enabled(const struct vcpu *v);
diff --git a/xen/include/asm-x86/hypercall.h b/xen/include/asm-x86/hypercall.h
index 1cc2e37d5c..da38b7991c 100644
--- a/xen/include/asm-x86/hypercall.h
+++ b/xen/include/asm-x86/hypercall.h
@@ -165,7 +165,7 @@ extern int compat_update_va_mapping(
unsigned int va, u32 lo, u32 hi, unsigned int flags);
extern int compat_update_va_mapping_otherdomain(
- unsigned long va, u32 lo, u32 hi, unsigned long flags, domid_t domid);
+ unsigned int va, u32 lo, u32 hi, unsigned int flags, domid_t domid);
DEFINE_XEN_GUEST_HANDLE(trap_info_compat_t);
extern int compat_set_trap_table(XEN_GUEST_HANDLE(trap_info_compat_t) traps);
diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
index 8fbccc88a7..7235623c86 100644
--- a/xen/include/asm-x86/msr-index.h
+++ b/xen/include/asm-x86/msr-index.h
@@ -47,8 +47,12 @@
#define ARCH_CAPABILITIES_RDCL_NO (_AC(1, ULL) << 0)
#define ARCH_CAPABILITIES_IBRS_ALL (_AC(1, ULL) << 1)
#define ARCH_CAPS_RSBA (_AC(1, ULL) << 2)
+#define ARCH_CAPS_SKIP_L1DFL (_AC(1, ULL) << 3)
#define ARCH_CAPS_SSB_NO (_AC(1, ULL) << 4)
+#define MSR_FLUSH_CMD 0x0000010b
+#define FLUSH_CMD_L1D (_AC(1, ULL) << 0)
+
/* Intel MSRs. Some also available on other CPUs */
#define MSR_IA32_PERFCTR0 0x000000c1
#define MSR_IA32_A_PERFCTR0 0x000004c1
diff --git a/xen/include/asm-x86/msr.h b/xen/include/asm-x86/msr.h
index f14f265aa5..afbeb7f155 100644
--- a/xen/include/asm-x86/msr.h
+++ b/xen/include/asm-x86/msr.h
@@ -241,7 +241,7 @@ static inline void write_efer(uint64_t val)
wrmsrl(MSR_EFER, val);
}
-DECLARE_PER_CPU(u32, ler_msr);
+extern unsigned int ler_msr;
DECLARE_PER_CPU(uint32_t, tsc_aux);
diff --git a/xen/include/asm-x86/paging.h b/xen/include/asm-x86/paging.h
index f0085511c7..f440e3e53c 100644
--- a/xen/include/asm-x86/paging.h
+++ b/xen/include/asm-x86/paging.h
@@ -37,11 +37,14 @@
#define PG_SH_shift 20
#define PG_HAP_shift 21
+#define PG_SHF_shift 22
/* We're in one of the shadow modes */
#ifdef CONFIG_SHADOW_PAGING
#define PG_SH_enable (1U << PG_SH_shift)
+#define PG_SH_forced (1U << PG_SHF_shift)
#else
#define PG_SH_enable 0
+#define PG_SH_forced 0
#endif
#define PG_HAP_enable (1U << PG_HAP_shift)
@@ -62,6 +65,7 @@
#define paging_mode_enabled(_d) (!!(_d)->arch.paging.mode)
#define paging_mode_shadow(_d) (!!((_d)->arch.paging.mode & PG_SH_enable))
+#define paging_mode_sh_forced(_d) (!!((_d)->arch.paging.mode & PG_SH_forced))
#define paging_mode_hap(_d) (!!((_d)->arch.paging.mode & PG_HAP_enable))
#define paging_mode_refcounts(_d) (!!((_d)->arch.paging.mode & PG_refcounts))
diff --git a/xen/include/asm-x86/processor.h b/xen/include/asm-x86/processor.h
index 9924cdf1f3..2bd9e69684 100644
--- a/xen/include/asm-x86/processor.h
+++ b/xen/include/asm-x86/processor.h
@@ -337,12 +337,6 @@ static always_inline void set_in_cr4 (unsigned long mask)
write_cr4(read_cr4() | mask);
}
-static always_inline void clear_in_cr4 (unsigned long mask)
-{
- mmu_cr4_features &= ~mask;
- write_cr4(read_cr4() & ~mask);
-}
-
static inline unsigned int read_pkru(void)
{
unsigned int pkru;
diff --git a/xen/include/asm-x86/setup.h b/xen/include/asm-x86/setup.h
index 19232afa01..c09a5ff381 100644
--- a/xen/include/asm-x86/setup.h
+++ b/xen/include/asm-x86/setup.h
@@ -66,6 +66,8 @@ extern uint8_t kbd_shift_flags;
extern unsigned long highmem_start;
#endif
+extern int8_t opt_smt;
+
#ifdef CONFIG_SHADOW_PAGING
extern bool opt_dom0_shadow;
#else
diff --git a/xen/include/asm-x86/shadow.h b/xen/include/asm-x86/shadow.h
index 94a34fd16a..f40f411871 100644
--- a/xen/include/asm-x86/shadow.h
+++ b/xen/include/asm-x86/shadow.h
@@ -29,6 +29,7 @@
#include <asm/flushtlb.h>
#include <asm/paging.h>
#include <asm/p2m.h>
+#include <asm/spec_ctrl.h>
/*****************************************************************************
* Macros to tell which shadow paging mode a domain is in*/
@@ -115,6 +116,131 @@ static inline int shadow_domctl(struct domain *d,
#endif /* CONFIG_SHADOW_PAGING */
+/*
+ * Mitigations for L1TF / CVE-2018-3620 for PV guests.
+ *
+ * We cannot alter an architecturally-legitimate PTE which a PV guest has
+ * chosen to write, as traditional paged-out metadata is L1TF-vulnerable.
+ * What we can do is force a PV guest which writes a vulnerable PTE into
+ * shadow mode, so Xen controls the pagetables which are reachable by the CPU
+ * pagewalk.
+ *
+ * The core of the L1TF vulnerability is that the address bits of the PTE
+ * (accounting for PSE and factoring in the level-relevant part of the linear
+ * access) are sent for an L1D lookup (to retrieve the next-level PTE, or
+ * eventual memory address) before the Present or reserved bits (which would
+ * cause a terminal fault) are accounted for. If an L1D hit occurs, the
+ * resulting data is available for potentially dependent instructions.
+ *
+ * For Present PTEs, the PV type-count safety logic ensures that the address
+ * bits always point at a guest-accessible frame, which is safe WRT L1TF from
+ * Xen's point of view. In practice, a PV guest should be unable to set any
+ * reserved bits, so should be unable to create any present L1TF-vulnerable
+ * PTEs at all.
+ *
+ * Therefore, these safety checks apply to Not-Present PTEs only, where
+ * traditionally, Xen would have let the guest write any value it chose.
+ *
+ * The all-zero PTE potentially leaks mfn 0. All software on the system is
+ * expected to cooperate and not put any secrets there. In a Xen system,
+ * neither Xen nor dom0 are expected to touch mfn 0, as it typically contains
+ * the real mode IVT and Bios Data Area. Therefore, mfn 0 is considered safe.
+ *
+ * Any PTE whose address is higher than the maximum cacheable address is safe,
+ * as it won't get an L1D hit.
+ *
+ * Speculative superpages also need accounting for, as PSE is considered
+ * irrespective of Present. We disallow PSE being set, as it allows an
+ * attacker to leak 2M or 1G of data starting from mfn 0. Also, because of
+ * recursive/linear pagetables, we must consider PSE even at L4, as hardware
+ * will interpret an L4e as an L3e during a recursive walk.
+ */
+
+static inline bool is_l1tf_safe_maddr(intpte_t pte)
+{
+ paddr_t maddr = pte & l1tf_addr_mask;
+
+ return maddr == 0 || maddr >= l1tf_safe_maddr;
+}
+
+static inline bool pv_l1tf_check_pte(struct domain *d, unsigned int level,
+ intpte_t pte)
+{
+ ASSERT(is_pv_domain(d));
+ ASSERT(!(pte & _PAGE_PRESENT));
+
+ if ( d->arch.pv_domain.check_l1tf && !paging_mode_sh_forced(d) &&
+ (((level > 1) && (pte & _PAGE_PSE)) || !is_l1tf_safe_maddr(pte)) )
+ {
+#ifdef CONFIG_SHADOW_PAGING
+ struct tasklet *t = &d->arch.paging.shadow.pv_l1tf_tasklet;
+
+ printk(XENLOG_G_WARNING
+ "d%d L1TF-vulnerable L%ue %016"PRIx64" - Shadowing\n",
+ d->domain_id, level, pte);
+ /*
+ * Safety consideration for accessing tasklet.scheduled_on without the
+ * tasklet lock. This is a singleshot tasklet with the side effect of
+ * setting PG_SH_forced (checked just above). Multiple vcpus can race
+ * to schedule the tasklet, but if we observe it scheduled anywhere,
+ * that is good enough.
+ */
+ smp_rmb();
+ if ( !tasklet_is_scheduled(t) )
+ tasklet_schedule(t);
+#else
+ printk(XENLOG_G_ERR
+ "d%d L1TF-vulnerable L%ue %016"PRIx64" - Crashing\n",
+ d->domain_id, level, pte);
+ domain_crash(d);
+#endif
+ return true;
+ }
+
+ return false;
+}
+
+static inline bool pv_l1tf_check_l1e(struct domain *d, l1_pgentry_t l1e)
+{
+ return pv_l1tf_check_pte(d, 1, l1e.l1);
+}
+
+static inline bool pv_l1tf_check_l2e(struct domain *d, l2_pgentry_t l2e)
+{
+ return pv_l1tf_check_pte(d, 2, l2e.l2);
+}
+
+static inline bool pv_l1tf_check_l3e(struct domain *d, l3_pgentry_t l3e)
+{
+ return pv_l1tf_check_pte(d, 3, l3e.l3);
+}
+
+static inline bool pv_l1tf_check_l4e(struct domain *d, l4_pgentry_t l4e)
+{
+ return pv_l1tf_check_pte(d, 4, l4e.l4);
+}
+
+void pv_l1tf_tasklet(unsigned long data);
+
+static inline void pv_l1tf_domain_init(struct domain *d)
+{
+ d->arch.pv_domain.check_l1tf =
+ opt_pv_l1tf & (is_hardware_domain(d)
+ ? OPT_PV_L1TF_DOM0 : OPT_PV_L1TF_DOMU);
+
+#if defined(CONFIG_SHADOW_PAGING) && defined(CONFIG_PV)
+ tasklet_init(&d->arch.paging.shadow.pv_l1tf_tasklet,
+ pv_l1tf_tasklet, (unsigned long)d);
+#endif
+}
+
+static inline void pv_l1tf_domain_destroy(struct domain *d)
+{
+#if defined(CONFIG_SHADOW_PAGING) && defined(CONFIG_PV)
+ tasklet_kill(&d->arch.paging.shadow.pv_l1tf_tasklet);
+#endif
+}
+
/* Remove all shadows of the guest mfn. */
static inline void shadow_remove_all_shadows(struct domain *d, mfn_t gmfn)
{
diff --git a/xen/include/asm-x86/smp.h b/xen/include/asm-x86/smp.h
index 4e5f673fec..09c55458df 100644
--- a/xen/include/asm-x86/smp.h
+++ b/xen/include/asm-x86/smp.h
@@ -26,6 +26,8 @@ DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_mask);
DECLARE_PER_CPU(cpumask_var_t, cpu_core_mask);
DECLARE_PER_CPU(cpumask_var_t, scratch_cpumask);
+extern bool park_offline_cpus;
+
void smp_send_nmi_allbutself(void);
void send_IPI_mask(const cpumask_t *, int vector);
diff --git a/xen/include/asm-x86/spec_ctrl.h b/xen/include/asm-x86/spec_ctrl.h
index 5b40afbab0..8f8aad40bb 100644
--- a/xen/include/asm-x86/spec_ctrl.h
+++ b/xen/include/asm-x86/spec_ctrl.h
@@ -29,15 +29,27 @@ void init_speculation_mitigations(void);
extern bool opt_ibpb;
extern bool opt_ssbd;
extern int8_t opt_eager_fpu;
+extern int8_t opt_l1d_flush;
extern bool bsp_delay_spec_ctrl;
extern uint8_t default_xen_spec_ctrl;
extern uint8_t default_spec_ctrl_flags;
-extern uint8_t opt_xpti;
+extern int8_t opt_xpti;
#define OPT_XPTI_DOM0 0x01
#define OPT_XPTI_DOMU 0x02
+extern int8_t opt_pv_l1tf;
+#define OPT_PV_L1TF_DOM0 0x01
+#define OPT_PV_L1TF_DOMU 0x02
+
+/*
+ * The L1D address mask, which might be wider than reported in CPUID, and the
+ * system physical address above which there are believed to be no cacheable
+ * memory regions, thus unable to leak data via the L1TF vulnerability.
+ */
+extern paddr_t l1tf_addr_mask, l1tf_safe_maddr;
+
static inline void init_shadow_spec_ctrl_state(void)
{
struct cpu_info *info = get_cpu_info();
diff --git a/xen/include/asm-x86/system.h b/xen/include/asm-x86/system.h
index 43fb6fe489..483cd20afd 100644
--- a/xen/include/asm-x86/system.h
+++ b/xen/include/asm-x86/system.h
@@ -221,6 +221,30 @@ static always_inline unsigned long __xadd(
#define set_mb(var, value) do { xchg(&var, value); } while (0)
#define set_wmb(var, value) do { var = value; smp_wmb(); } while (0)
+/**
+ * array_index_mask_nospec() - generate a mask that is ~0UL when the
+ * bounds check succeeds and 0 otherwise
+ * @index: array element index
+ * @size: number of elements in array
+ *
+ * Returns:
+ * 0 - (index < size)
+ */
+static inline unsigned long array_index_mask_nospec(unsigned long index,
+ unsigned long size)
+{
+ unsigned long mask;
+
+ asm volatile ( "cmp %[size], %[index]; sbb %[mask], %[mask];"
+ : [mask] "=r" (mask)
+ : [size] "g" (size), [index] "r" (index) );
+
+ return mask;
+}
+
+/* Override default implementation in nospec.h. */
+#define array_index_mask_nospec array_index_mask_nospec
+
#define local_irq_disable() asm volatile ( "cli" : : : "memory" )
#define local_irq_enable() asm volatile ( "sti" : : : "memory" )
diff --git a/xen/include/asm-x86/xstate.h b/xen/include/asm-x86/xstate.h
index 86a4a1f75c..47f602b855 100644
--- a/xen/include/asm-x86/xstate.h
+++ b/xen/include/asm-x86/xstate.h
@@ -97,8 +97,9 @@ void xsave(struct vcpu *v, uint64_t mask);
void xrstor(struct vcpu *v, uint64_t mask);
void xstate_set_init(uint64_t mask);
bool xsave_enabled(const struct vcpu *v);
-int __must_check validate_xstate(u64 xcr0, u64 xcr0_accum,
- const struct xsave_hdr *);
+int __must_check validate_xstate(const struct domain *d,
+ uint64_t xcr0, uint64_t xcr0_accum,
+ const struct xsave_hdr *hdr);
int __must_check handle_xsetbv(u32 index, u64 new_bv);
void expand_xsave_states(struct vcpu *v, void *dest, unsigned int size);
void compress_xsave_states(struct vcpu *v, const void *src, unsigned int size);
diff --git a/xen/include/public/arch-x86/cpufeatureset.h b/xen/include/public/arch-x86/cpufeatureset.h
index f1a5ed93e0..6c82816fd3 100644
--- a/xen/include/public/arch-x86/cpufeatureset.h
+++ b/xen/include/public/arch-x86/cpufeatureset.h
@@ -244,6 +244,7 @@ XEN_CPUFEATURE(AVX512_4VNNIW, 9*32+ 2) /*A AVX512 Neural Network Instructions *
XEN_CPUFEATURE(AVX512_4FMAPS, 9*32+ 3) /*A AVX512 Multiply Accumulation Single Precision */
XEN_CPUFEATURE(IBRSB, 9*32+26) /*A IBRS and IBPB support (used by Intel) */
XEN_CPUFEATURE(STIBP, 9*32+27) /*A STIBP */
+XEN_CPUFEATURE(L1D_FLUSH, 9*32+28) /*S MSR_FLUSH_CMD and L1D flush. */
XEN_CPUFEATURE(ARCH_CAPS, 9*32+29) /* IA32_ARCH_CAPABILITIES MSR */
XEN_CPUFEATURE(SSBD, 9*32+31) /*A MSR_SPEC_CTRL.SSBD available */
diff --git a/xen/include/xen/compiler.h b/xen/include/xen/compiler.h
index 533a8ea0f3..a7e05681c9 100644
--- a/xen/include/xen/compiler.h
+++ b/xen/include/xen/compiler.h
@@ -81,6 +81,9 @@
#pragma GCC visibility push(hidden)
#endif
+/* Make the optimizer believe the variable can be manipulated arbitrarily. */
+#define OPTIMIZER_HIDE_VAR(var) __asm__ ( "" : "+g" (var) )
+
/* This macro obfuscates arithmetic on a variable address so that gcc
shouldn't recognize the original var, and make assumptions about it */
/*
diff --git a/xen/include/xen/cpu.h b/xen/include/xen/cpu.h
index ffefc09f8e..2fe3ec05d8 100644
--- a/xen/include/xen/cpu.h
+++ b/xen/include/xen/cpu.h
@@ -47,6 +47,8 @@ void register_cpu_notifier(struct notifier_block *nb);
#define CPU_DYING (0x0007 | NOTIFY_REVERSE)
/* CPU_DEAD: CPU is dead. */
#define CPU_DEAD (0x0008 | NOTIFY_REVERSE)
+/* CPU_REMOVE: CPU was removed. */
+#define CPU_REMOVE (0x0009 | NOTIFY_REVERSE)
/* Perform CPU hotplug. May return -EAGAIN. */
int cpu_down(unsigned int cpu);
diff --git a/xen/include/xen/cpumask.h b/xen/include/xen/cpumask.h
index 42340a098e..4a11bcc3f3 100644
--- a/xen/include/xen/cpumask.h
+++ b/xen/include/xen/cpumask.h
@@ -351,16 +351,35 @@ static inline bool_t alloc_cpumask_var(cpumask_var_t *mask)
return *mask != NULL;
}
+static inline bool cond_alloc_cpumask_var(cpumask_var_t *mask)
+{
+ if (*mask == NULL)
+ *mask = _xmalloc(nr_cpumask_bits / 8, sizeof(long));
+ return *mask != NULL;
+}
+
static inline bool_t zalloc_cpumask_var(cpumask_var_t *mask)
{
*(void **)mask = _xzalloc(nr_cpumask_bits / 8, sizeof(long));
return *mask != NULL;
}
+static inline bool cond_zalloc_cpumask_var(cpumask_var_t *mask)
+{
+ if (*mask == NULL)
+ *mask = _xzalloc(nr_cpumask_bits / 8, sizeof(long));
+ else
+ cpumask_clear(*mask);
+ return *mask != NULL;
+}
+
static inline void free_cpumask_var(cpumask_var_t mask)
{
xfree(mask);
}
+
+/* Free an allocated mask, and zero the pointer to it. */
+#define FREE_CPUMASK_VAR(m) XFREE(m)
#else
typedef cpumask_t cpumask_var_t[1];
@@ -368,16 +387,20 @@ static inline bool_t alloc_cpumask_var(cpumask_var_t *mask)
{
return 1;
}
+#define cond_alloc_cpumask_var alloc_cpumask_var
static inline bool_t zalloc_cpumask_var(cpumask_var_t *mask)
{
cpumask_clear(*mask);
return 1;
}
+#define cond_zalloc_cpumask_var zalloc_cpumask_var
static inline void free_cpumask_var(cpumask_var_t mask)
{
}
+
+#define FREE_CPUMASK_VAR(m) free_cpumask_var(m)
#endif
#if NR_CPUS > 1
diff --git a/xen/include/xen/list.h b/xen/include/xen/list.h
index fa07d720ee..1387abb211 100644
--- a/xen/include/xen/list.h
+++ b/xen/include/xen/list.h
@@ -51,6 +51,11 @@ static inline void INIT_LIST_HEAD(struct list_head *list)
list->prev = list;
}
+static inline bool list_head_is_null(const struct list_head *list)
+{
+ return !list->next && !list->prev;
+}
+
/*
* Insert a new entry between two known consecutive entries.
*
diff --git a/xen/include/xen/mm.h b/xen/include/xen/mm.h
index e928551c91..24654e8e22 100644
--- a/xen/include/xen/mm.h
+++ b/xen/include/xen/mm.h
@@ -162,6 +162,14 @@ void free_xenheap_pages(void *v, unsigned int order);
bool scrub_free_pages(void);
#define alloc_xenheap_page() (alloc_xenheap_pages(0,0))
#define free_xenheap_page(v) (free_xenheap_pages(v,0))
+
+/* Free an allocation, and zero the pointer to it. */
+#define FREE_XENHEAP_PAGES(p, o) do { \
+ free_xenheap_pages(p, o); \
+ (p) = NULL; \
+} while ( false )
+#define FREE_XENHEAP_PAGE(p) FREE_XENHEAP_PAGES(p, 0)
+
/* Map machine page range in Xen virtual address space. */
int map_pages_to_xen(
unsigned long virt,
diff --git a/xen/include/xen/nospec.h b/xen/include/xen/nospec.h
new file mode 100644
index 0000000000..48793996e8
--- /dev/null
+++ b/xen/include/xen/nospec.h
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2018 Linus Torvalds. All rights reserved. */
+/* Copyright(c) 2018 Alexei Starovoitov. All rights reserved. */
+/* Copyright(c) 2018 Intel Corporation. All rights reserved. */
+/* Copyright(c) 2018 Citrix Systems R&D Ltd. All rights reserved. */
+
+#ifndef XEN_NOSPEC_H
+#define XEN_NOSPEC_H
+
+#include <asm/system.h>
+
+/**
+ * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise
+ * @index: array element index
+ * @size: number of elements in array
+ *
+ * When @index is out of bounds (@index >= @size), the sign bit will be
+ * set. Extend the sign bit to all bits and invert, giving a result of
+ * zero for an out of bounds index, or ~0 if within bounds [0, @size).
+ */
+#ifndef array_index_mask_nospec
+static inline unsigned long array_index_mask_nospec(unsigned long index,
+ unsigned long size)
+{
+ /*
+ * Always calculate and emit the mask even if the compiler
+ * thinks the mask is not needed. The compiler does not take
+ * into account the value of @index under speculation.
+ */
+ OPTIMIZER_HIDE_VAR(index);
+ return ~(long)(index | (size - 1UL - index)) >> (BITS_PER_LONG - 1);
+}
+#endif
+
+/*
+ * array_index_nospec - sanitize an array index after a bounds check
+ *
+ * For a code sequence like:
+ *
+ * if (index < size) {
+ * index = array_index_nospec(index, size);
+ * val = array[index];
+ * }
+ *
+ * ...if the CPU speculates past the bounds check then
+ * array_index_nospec() will clamp the index within the range of [0,
+ * size).
+ */
+#define array_index_nospec(index, size) \
+({ \
+ typeof(index) _i = (index); \
+ typeof(size) _s = (size); \
+ unsigned long _mask = array_index_mask_nospec(_i, _s); \
+ \
+ BUILD_BUG_ON(sizeof(_i) > sizeof(long)); \
+ BUILD_BUG_ON(sizeof(_s) > sizeof(long)); \
+ \
+ (typeof(_i)) (_i & _mask); \
+})
+
+#endif /* XEN_NOSPEC_H */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
index 99d2af2e1f..e79d5a36ca 100644
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -788,7 +788,7 @@ static inline struct domain *next_domain_in_cpupool(
#define _VPF_parked 8
#define VPF_parked (1UL<<_VPF_parked)
-static inline int vcpu_runnable(struct vcpu *v)
+static inline bool vcpu_runnable(const struct vcpu *v)
{
return !(v->pause_flags |
atomic_read(&v->pause_count) |
diff --git a/xen/include/xen/tasklet.h b/xen/include/xen/tasklet.h
index 23d69c738e..bc9ddace6d 100644
--- a/xen/include/xen/tasklet.h
+++ b/xen/include/xen/tasklet.h
@@ -50,6 +50,11 @@ static inline bool tasklet_work_to_do(unsigned int cpu)
TASKLET_scheduled);
}
+static inline bool tasklet_is_scheduled(const struct tasklet *t)
+{
+ return t->scheduled_on != -1;
+}
+
void tasklet_schedule_on_cpu(struct tasklet *t, unsigned int cpu);
void tasklet_schedule(struct tasklet *t);
void do_tasklet(void);
diff --git a/xen/include/xen/xmalloc.h b/xen/include/xen/xmalloc.h
index cc2673d8ae..9aa5edf593 100644
--- a/xen/include/xen/xmalloc.h
+++ b/xen/include/xen/xmalloc.h
@@ -26,6 +26,12 @@
/* Free any of the above. */
extern void xfree(void *);
+/* Free an allocation, and zero the pointer to it. */
+#define XFREE(p) do { \
+ xfree(p); \
+ (p) = NULL; \
+} while ( false )
+
/* Underlying functions */
extern void *_xmalloc(unsigned long size, unsigned long align);
extern void *_xzalloc(unsigned long size, unsigned long align);
Reference in a new issue View git blame Copy permalink