diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index a7ec57060f64..4c300caad901 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -212,6 +212,7 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | GIC-700 | #2941627 | ARM64_ERRATUM_2941627 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | SI L1 | #4311569 | ARM64_ERRATUM_4311569 | +----------------+-----------------+-----------------+-----------------------------+ | Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_845719 | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 93173f0a09c7..89326bb26f48 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1155,6 +1155,25 @@ config ARM64_ERRATUM_3194386 If unsure, say Y. +config ARM64_ERRATUM_4311569 + bool "SI L1: 4311569: workaround for premature CMO completion erratum" + default y + help + This option adds the workaround for ARM SI L1 erratum 4311569. + + The erratum of SI L1 can cause an early response to a combined write + and cache maintenance operation (WR+CMO) before the operation is fully + completed to the Point of Serialization (POS). + This can result in a non-I/O coherent agent observing stale data, + potentially leading to system instability or incorrect behavior. + + Enabling this option implements a software workaround by inserting a + second loop of Cache Maintenance Operation (CMO) immediately following the + end of function to do CMOs. This ensures that the data is correctly serialized + before the buffer is handed off to a non-coherent agent. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index f0ca7196f6fa..d3d46e5f7188 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -381,6 +381,9 @@ alternative_endif .macro dcache_by_myline_op op, domain, start, end, linesz, tmp, fixup sub \tmp, \linesz, #1 bic \start, \start, \tmp +alternative_if ARM64_WORKAROUND_4311569 + mov \tmp, \start +alternative_else_nop_endif .Ldcache_op\@: .ifc \op, cvau __dcache_op_workaround_clean_cache \op, \start @@ -402,6 +405,13 @@ alternative_endif add \start, \start, \linesz cmp \start, \end b.lo .Ldcache_op\@ +alternative_if ARM64_WORKAROUND_4311569 + .ifnc \op, cvau + mov \start, \tmp + mov \tmp, xzr + cbnz \start, .Ldcache_op\@ + .endif +alternative_else_nop_endif dsb \domain _cond_uaccess_extable .Ldcache_op\@, \fixup diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 8cb3b575a031..5c0ab6bfd44a 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -141,6 +141,30 @@ has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry, return (ctr_real != sys) && (ctr_raw != sys); } +#ifdef CONFIG_ARM64_ERRATUM_4311569 +static DEFINE_STATIC_KEY_FALSE(arm_si_l1_workaround_4311569); +static int __init early_arm_si_l1_workaround_4311569_cfg(char *arg) +{ + static_branch_enable(&arm_si_l1_workaround_4311569); + pr_info("Enabling cache maintenance workaround for ARM SI-L1 erratum 4311569\n"); + + return 0; +} +early_param("arm_si_l1_workaround_4311569", early_arm_si_l1_workaround_4311569_cfg); + +/* + * We have some earlier use cases to call cache maintenance operation functions, for example, + * dcache_inval_poc() and dcache_clean_poc() in head.S, before making decision to turn on this + * workaround. Since the scope of this workaround is limited to non-coherent DMA agents, its + * safe to have the workaround off by default. + */ +static bool +need_arm_si_l1_workaround_4311569(const struct arm64_cpu_capabilities *entry, int scope) +{ + return static_branch_unlikely(&arm_si_l1_workaround_4311569); +} +#endif + static void cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *cap) { @@ -870,6 +894,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ERRATA_MIDR_RANGE_LIST(erratum_spec_ssbs_list), }, #endif +#ifdef CONFIG_ARM64_ERRATUM_4311569 + { + .capability = ARM64_WORKAROUND_4311569, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = need_arm_si_l1_workaround_4311569, + }, +#endif #ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD { .desc = "ARM errata 2966298, 3117295", diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 0fac75f01534..856b6cf6e71e 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -103,6 +103,7 @@ WORKAROUND_2077057 WORKAROUND_2457168 WORKAROUND_2645198 WORKAROUND_2658417 +WORKAROUND_4311569 WORKAROUND_AMPERE_AC03_CPU_38 WORKAROUND_AMPERE_AC04_CPU_23 WORKAROUND_TRBE_OVERWRITE_FILL_MODE