diff options
774 files changed, 21218 insertions, 6892 deletions
diff --git a/Documentation/ABI/obsolete/sysfs-class-dax b/Documentation/ABI/obsolete/sysfs-class-dax new file mode 100644 index 0000000000000..2cb9fc5e8bd14 --- /dev/null +++ b/Documentation/ABI/obsolete/sysfs-class-dax @@ -0,0 +1,22 @@ +What: /sys/class/dax/ +Date: May, 2016 +KernelVersion: v4.7 +Contact: linux-nvdimm@lists.01.org +Description: Device DAX is the device-centric analogue of Filesystem + DAX (CONFIG_FS_DAX). It allows memory ranges to be + allocated and mapped without need of an intervening file + system. Device DAX is strict, precise and predictable. + Specifically this interface: + + 1/ Guarantees fault granularity with respect to a given + page size (pte, pmd, or pud) set at configuration time. + + 2/ Enforces deterministic behavior by being strict about + what fault scenarios are supported. + + The /sys/class/dax/ interface enumerates all the + device-dax instances in the system. The ABI is + deprecated and will be removed after 2020. It is + replaced with the DAX bus interface /sys/bus/dax/ where + device-dax instances can be found under + /sys/bus/dax/devices/ diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index a7ce331994578..91822ce258317 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -86,6 +86,13 @@ Description: The unit size is one block, now only support configuring in range of [1, 512]. +What: /sys/fs/f2fs/<disk>/umount_discard_timeout +Date: January 2019 +Contact: "Jaegeuk Kim" <jaegeuk@kernel.org> +Description: + Set timeout to issue discard commands during umount. + Default: 5 secs + What: /sys/fs/f2fs/<disk>/max_victim_search Date: January 2014 Contact: "Jaegeuk Kim" <jaegeuk.kim@samsung.com> diff --git a/Documentation/acpi/aml-debugger.txt b/Documentation/acpi/aml-debugger.txt index e851cc5de63f2..75ebeb64ab29a 100644 --- a/Documentation/acpi/aml-debugger.txt +++ b/Documentation/acpi/aml-debugger.txt @@ -23,7 +23,7 @@ kernel. The resultant userspace tool binary is then located at: - tools/acpi/power/acpi/acpidbg/acpidbg + tools/power/acpi/acpidbg It can be installed to system directories by running "make install" (as a sufficiently privileged user). @@ -35,7 +35,7 @@ kernel. # mount -t debugfs none /sys/kernel/debug # modprobe acpi_dbg - # tools/acpi/power/acpi/acpidbg/acpidbg + # tools/power/acpi/acpidbg That spawns the interactive AML debugger environment where you can execute debugger commands. diff --git a/Documentation/admin-guide/md.rst b/Documentation/admin-guide/md.rst index 84de718f24a46..3c51084ffd379 100644 --- a/Documentation/admin-guide/md.rst +++ b/Documentation/admin-guide/md.rst @@ -756,3 +756,6 @@ These currently include: The cache mode for raid5. raid5 could include an extra disk for caching. The mode can be "write-throuth" and "write-back". The default is "write-through". + + ppl_write_hint + NVMe stream ID to be set for each PPL write request. diff --git a/Documentation/arm/kernel_mode_neon.txt b/Documentation/arm/kernel_mode_neon.txt index 525452726d31e..b9e060c5b61e0 100644 --- a/Documentation/arm/kernel_mode_neon.txt +++ b/Documentation/arm/kernel_mode_neon.txt @@ -6,7 +6,7 @@ TL;DR summary * Use only NEON instructions, or VFP instructions that don't rely on support code * Isolate your NEON code in a separate compilation unit, and compile it with - '-mfpu=neon -mfloat-abi=softfp' + '-march=armv7-a -mfpu=neon -mfloat-abi=softfp' * Put kernel_neon_begin() and kernel_neon_end() calls around the calls into your NEON code * Don't sleep in your NEON code, and be aware that it will be executed with @@ -87,7 +87,7 @@ instructions appearing in unexpected places if no special care is taken. Therefore, the recommended and only supported way of using NEON/VFP in the kernel is by adhering to the following rules: * isolate the NEON code in a separate compilation unit and compile it with - '-mfpu=neon -mfloat-abi=softfp'; + '-march=armv7-a -mfpu=neon -mfloat-abi=softfp'; * issue the calls to kernel_neon_begin(), kernel_neon_end() as well as the calls into the unit containing the NEON code from a compilation unit which is *not* built with the GCC flag '-mfpu=neon' set. diff --git a/Documentation/devicetree/bindings/clock/actions,owl-cmu.txt b/Documentation/devicetree/bindings/clock/actions,owl-cmu.txt index 2ef86ae96df8c..d19885b7c73fc 100644 --- a/Documentation/devicetree/bindings/clock/actions,owl-cmu.txt +++ b/Documentation/devicetree/bindings/clock/actions,owl-cmu.txt @@ -2,13 +2,14 @@ The Actions Semi Owl Clock Management Unit generates and supplies clock to various controllers within the SoC. The clock binding described here is -applicable to S900 and S700 SoC's. +applicable to S900, S700 and S500 SoC's. Required Properties: - compatible: should be one of the following, "actions,s900-cmu" "actions,s700-cmu" + "actions,s500-cmu" - reg: physical base address of the controller and length of memory mapped region. - clocks: Reference to the parent clocks ("hosc", "losc") @@ -19,8 +20,8 @@ Each clock is assigned an identifier, and client nodes can use this identifier to specify the clock which they consume. All available clocks are defined as preprocessor macros in corresponding -dt-bindings/clock/actions,s900-cmu.h or actions,s700-cmu.h header and can be -used in device tree sources. +dt-bindings/clock/actions,s900-cmu.h or actions,s700-cmu.h or +actions,s500-cmu.h header and can be used in device tree sources. External clocks: diff --git a/Documentation/devicetree/bindings/clock/amlogic,gxbb-aoclkc.txt b/Documentation/devicetree/bindings/clock/amlogic,gxbb-aoclkc.txt index 79511d7bb3218..c41f0be5d4384 100644 --- a/Documentation/devicetree/bindings/clock/amlogic,gxbb-aoclkc.txt +++ b/Documentation/devicetree/bindings/clock/amlogic,gxbb-aoclkc.txt @@ -10,6 +10,7 @@ Required Properties: - GXL (S905X, S905D) : "amlogic,meson-gxl-aoclkc" - GXM (S912) : "amlogic,meson-gxm-aoclkc" - AXG (A113D, A113X) : "amlogic,meson-axg-aoclkc" + - G12A (S905X2, S905D2, S905Y2) : "amlogic,meson-g12a-aoclkc" followed by the common "amlogic,meson-gx-aoclkc" - clocks: list of clock phandle, one for each entry clock-names. - clock-names: should contain the following: diff --git a/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt b/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt index a6871953bf044..5c8b105be4d66 100644 --- a/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt +++ b/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt @@ -9,6 +9,7 @@ Required Properties: "amlogic,gxbb-clkc" for GXBB SoC, "amlogic,gxl-clkc" for GXL and GXM SoC, "amlogic,axg-clkc" for AXG SoC. + "amlogic,g12a-clkc" for G12A SoC. - clocks : list of clock phandle, one for each entry clock-names. - clock-names : should contain the following: * "xtal": the platform xtal diff --git a/Documentation/devicetree/bindings/clock/exynos5433-clock.txt b/Documentation/devicetree/bindings/clock/exynos5433-clock.txt index 50d5897c98491..183c327a7d6bd 100644 --- a/Documentation/devicetree/bindings/clock/exynos5433-clock.txt +++ b/Documentation/devicetree/bindings/clock/exynos5433-clock.txt @@ -50,6 +50,8 @@ Required Properties: IPs. - "samsung,exynos5433-cmu-cam1" - clock controller compatible for CMU_CAM1 which generates clocks for Cortex-A5/MIPI_CSIS2/FIMC-LITE_C/FIMC-FD IPs. + - "samsung,exynos5433-cmu-imem" - clock controller compatible for CMU_IMEM + which generates clocks for SSS (Security SubSystem) and SlimSSS IPs. - reg: physical base address of the controller and length of memory mapped region. @@ -168,6 +170,12 @@ Required Properties: - aclk_cam1_400 - aclk_cam1_552 + Input clocks for imem clock controller: + - oscclk + - aclk_imem_sssx_266 + - aclk_imem_266 + - aclk_imem_200 + Optional properties: - power-domains: a phandle to respective power domain node as described by generic PM domain bindings (see power/power_domain.txt for more @@ -469,6 +477,21 @@ Example 2: Examples of clock controller nodes are listed below. power-domains = <&pd_cam1>; }; + cmu_imem: clock-controller@11060000 { + compatible = "samsung,exynos5433-cmu-imem"; + reg = <0x11060000 0x1000>; + #clock-cells = <1>; + + clock-names = "oscclk", + "aclk_imem_sssx_266", + "aclk_imem_266", + "aclk_imem_200"; + clocks = <&xxti>, + <&cmu_top CLK_DIV_ACLK_IMEM_SSSX_266>, + <&cmu_top CLK_DIV_ACLK_IMEM_266>, + <&cmu_top CLK_DIV_ACLK_IMEM_200>; + }; + Example 3: UART controller node that consumes the clock generated by the clock controller. diff --git a/Documentation/devicetree/bindings/clock/fixed-clock.txt b/Documentation/devicetree/bindings/clock/fixed-clock.txt deleted file mode 100644 index 0641a663ad692..0000000000000 --- a/Documentation/devicetree/bindings/clock/fixed-clock.txt +++ /dev/null @@ -1,23 +0,0 @@ -Binding for simple fixed-rate clock sources. - -This binding uses the common clock binding[1]. - -[1] Documentation/devicetree/bindings/clock/clock-bindings.txt - -Required properties: -- compatible : shall be "fixed-clock". -- #clock-cells : from common clock binding; shall be set to 0. -- clock-frequency : frequency of clock in Hz. Should be a single cell. - -Optional properties: -- clock-accuracy : accuracy of clock in ppb (parts per billion). - Should be a single cell. -- clock-output-names : From common clock binding. - -Example: - clock { - compatible = "fixed-clock"; - #clock-cells = <0>; - clock-frequency = <1000000000>; - clock-accuracy = <100>; - }; diff --git a/Documentation/devicetree/bindings/clock/fixed-clock.yaml b/Documentation/devicetree/bindings/clock/fixed-clock.yaml new file mode 100644 index 0000000000000..b657ecd0ef1c3 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/fixed-clock.yaml @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/fixed-clock.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Binding for simple fixed-rate clock sources + +maintainers: + - Michael Turquette <mturquette@baylibre.com> + - Stephen Boyd <sboyd@kernel.org> + +properties: + compatible: + const: fixed-clock + + "#clock-cells": + const: 0 + + clock-frequency: true + + clock-accuracy: + description: accuracy of clock in ppb (parts per billion). + $ref: /schemas/types.yaml#/definitions/uint32 + + clock-output-names: + maxItems: 1 + +required: + - compatible + - "#clock-cells" + - clock-frequency + +additionalProperties: false + +examples: + - | + clock { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <1000000000>; + clock-accuracy = <100>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/fixed-factor-clock.txt b/Documentation/devicetree/bindings/clock/fixed-factor-clock.txt deleted file mode 100644 index 189467a7188af..0000000000000 --- a/Documentation/devicetree/bindings/clock/fixed-factor-clock.txt +++ /dev/null @@ -1,28 +0,0 @@ -Binding for simple fixed factor rate clock sources. - -This binding uses the common clock binding[1]. - -[1] Documentation/devicetree/bindings/clock/clock-bindings.txt - -Required properties: -- compatible : shall be "fixed-factor-clock". -- #clock-cells : from common clock binding; shall be set to 0. -- clock-div: fixed divider. -- clock-mult: fixed multiplier. -- clocks: parent clock. - -Optional properties: -- clock-output-names : From common clock binding. - -Some clocks that require special treatments are also handled by that -driver, with the compatibles: - - allwinner,sun4i-a10-pll3-2x-clk - -Example: - clock { - compatible = "fixed-factor-clock"; - clocks = <&parentclk>; - #clock-cells = <0>; - clock-div = <2>; - clock-mult = <1>; - }; diff --git a/Documentation/devicetree/bindings/clock/fixed-factor-clock.yaml b/Documentation/devicetree/bindings/clock/fixed-factor-clock.yaml new file mode 100644 index 0000000000000..b567f8092f8c8 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/fixed-factor-clock.yaml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/fixed-factor-clock.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Binding for simple fixed factor rate clock sources + +maintainers: + - Michael Turquette <mturquette@baylibre.com> + - Stephen Boyd <sboyd@kernel.org> + +properties: + compatible: + enum: + - allwinner,sun4i-a10-pll3-2x-clk + - fixed-factor-clock + + "#clock-cells": + const: 0 + + clocks: + maxItems: 1 + + clock-div: + description: Fixed divider + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + - minimum: 1 + + clock-mult: + description: Fixed multiplier + $ref: /schemas/types.yaml#/definitions/uint32 + + clock-output-names: + maxItems: 1 + +required: + - compatible + - clocks + - "#clock-cells" + - clock-div + - clock-mult + +additionalProperties: false + +examples: + - | + clock { + compatible = "fixed-factor-clock"; + clocks = <&parentclk>; + #clock-cells = <0>; + clock-div = <2>; + clock-mult = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/fixed-mmio-clock.txt b/Documentation/devicetree/bindings/clock/fixed-mmio-clock.txt new file mode 100644 index 0000000000000..c359367fd1a97 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/fixed-mmio-clock.txt @@ -0,0 +1,24 @@ +Binding for simple memory mapped io fixed-rate clock sources. +The driver reads a clock frequency value from a single 32-bit memory mapped +I/O register and registers it as a fixed rate clock. + +It was designed for test systems, like FPGA, not for complete, finished SoCs. + +This binding uses the common clock binding[1]. + +[1] Documentation/devicetree/bindings/clock/clock-bindings.txt + +Required properties: +- compatible : shall be "fixed-mmio-clock". +- #clock-cells : from common clock binding; shall be set to 0. +- reg : Address and length of the clock value register set. + +Optional properties: +- clock-output-names : From common clock binding. + +Example: +sysclock: sysclock@fd020004 { + #clock-cells = <0>; + compatible = "fixed-mmio-clock"; + reg = <0xfd020004 0x4>; +}; diff --git a/Documentation/devicetree/bindings/clock/imx8mm-clock.txt b/Documentation/devicetree/bindings/clock/imx8mm-clock.txt new file mode 100644 index 0000000000000..8e4ab9e619a10 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/imx8mm-clock.txt @@ -0,0 +1,29 @@ +* Clock bindings for NXP i.MX8M Mini + +Required properties: +- compatible: Should be "fsl,imx8mm-ccm" +- reg: Address and length of the register set +- #clock-cells: Should be <1> +- clocks: list of clock specifiers, must contain an entry for each required + entry in clock-names +- clock-names: should include the following entries: + - "osc_32k" + - "osc_24m" + - "clk_ext1" + - "clk_ext2" + - "clk_ext3" + - "clk_ext4" + +clk: clock-controller@30380000 { + compatible = "fsl,imx8mm-ccm"; + reg = <0x0 0x30380000 0x0 0x10000>; + #clock-cells = <1>; + clocks = <&osc_32k>, <&osc_24m>, <&clk_ext1>, <&clk_ext2>, + <&clk_ext3>, <&clk_ext4>; + clock-names = "osc_32k", "osc_24m", "clk_ext1", "clk_ext2", + "clk_ext3", "clk_ext4"; +}; + +The clock consumer should specify the desired clock by having the clock +ID in its "clocks" phandle cell. See include/dt-bindings/clock/imx8mm-clock.h +for the full list of i.MX8M Mini clock IDs. diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmcc.txt b/Documentation/devicetree/bindings/clock/qcom,rpmcc.txt index 87b4949e9bc8e..944719bd586f7 100644 --- a/Documentation/devicetree/bindings/clock/qcom,rpmcc.txt +++ b/Documentation/devicetree/bindings/clock/qcom,rpmcc.txt @@ -16,6 +16,7 @@ Required properties : "qcom,rpmcc-msm8974", "qcom,rpmcc" "qcom,rpmcc-apq8064", "qcom,rpmcc" "qcom,rpmcc-msm8996", "qcom,rpmcc" + "qcom,rpmcc-msm8998", "qcom,rpmcc" "qcom,rpmcc-qcs404", "qcom,rpmcc" - #clock-cells : shall contain 1 diff --git a/Documentation/devicetree/bindings/display/ssd1307fb.txt b/Documentation/devicetree/bindings/display/ssd1307fb.txt index 209d931ef16c4..b67f8caa212c8 100644 --- a/Documentation/devicetree/bindings/display/ssd1307fb.txt +++ b/Documentation/devicetree/bindings/display/ssd1307fb.txt @@ -36,7 +36,6 @@ ssd1307: oled@3c { reg = <0x3c>; pwms = <&pwm 4 3000>; reset-gpios = <&gpio2 7>; - reset-active-low; }; ssd1306: oled@3c { @@ -44,7 +43,6 @@ ssd1306: oled@3c { reg = <0x3c>; pwms = <&pwm 4 3000>; reset-gpios = <&gpio2 7>; - reset-active-low; solomon,com-lrremap; solomon,com-invdir; solomon,com-offset = <32>; diff --git a/Documentation/devicetree/bindings/dma/dma.txt b/Documentation/devicetree/bindings/dma/dma.txt index 6312fb00ce8d3..eeb4e4d1771ed 100644 --- a/Documentation/devicetree/bindings/dma/dma.txt +++ b/Documentation/devicetree/bindings/dma/dma.txt @@ -16,6 +16,9 @@ Optional properties: - dma-channels: Number of DMA channels supported by the controller. - dma-requests: Number of DMA request signals supported by the controller. +- dma-channel-mask: Bitmask of available DMA channels in ascending order + that are not reserved by firmware and are available to + the kernel. i.e. first channel corresponds to LSB. Example: @@ -29,6 +32,7 @@ Example: #dma-cells = <1>; dma-channels = <32>; dma-requests = <127>; + dma-channel-mask = <0xfffe> }; * DMA router diff --git a/Documentation/devicetree/bindings/dma/fsl-qdma.txt b/Documentation/devicetree/bindings/dma/fsl-qdma.txt new file mode 100644 index 0000000000000..6a0ff9059e72f --- /dev/null +++ b/Documentation/devicetree/bindings/dma/fsl-qdma.txt @@ -0,0 +1,57 @@ +NXP Layerscape SoC qDMA Controller +================================== + +This device follows the generic DMA bindings defined in dma/dma.txt. + +Required properties: + +- compatible: Must be one of + "fsl,ls1021a-qdma": for LS1021A Board + "fsl,ls1043a-qdma": for ls1043A Board + "fsl,ls1046a-qdma": for ls1046A Board +- reg: Should contain the register's base address and length. +- interrupts: Should contain a reference to the interrupt used by this + device. +- interrupt-names: Should contain interrupt names: + "qdma-queue0": the block0 interrupt + "qdma-queue1": the block1 interrupt + "qdma-queue2": the block2 interrupt + "qdma-queue3": the block3 interrupt + "qdma-error": the error interrupt +- fsl,dma-queues: Should contain number of queues supported. +- dma-channels: Number of DMA channels supported +- block-number: the virtual block number +- block-offset: the offset of different virtual block +- status-sizes: status queue size of per virtual block +- queue-sizes: command queue size of per virtual block, the size number + based on queues + +Optional properties: + +- dma-channels: Number of DMA channels supported by the controller. +- big-endian: If present registers and hardware scatter/gather descriptors + of the qDMA are implemented in big endian mode, otherwise in little + mode. + +Examples: + + qdma: dma-controller@8390000 { + compatible = "fsl,ls1021a-qdma"; + reg = <0x0 0x8388000 0x0 0x1000>, /* Controller regs */ + <0x0 0x8389000 0x0 0x1000>, /* Status regs */ + <0x0 0x838a000 0x0 0x2000>; /* Block regs */ + interrupts = <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "qdma-error", + "qdma-queue0", "qdma-queue1"; + dma-channels = <8>; + block-number = <2>; + block-offset = <0x1000>; + fsl,dma-queues = <2>; + status-sizes = <64>; + queue-sizes = <64 64>; + big-endian; + }; + +DMA clients must use the format described in dma/dma.txt file. diff --git a/Documentation/devicetree/bindings/dma/k3dma.txt b/Documentation/devicetree/bindings/dma/k3dma.txt index 4945aeac4dc4f..10a2f15b08a38 100644 --- a/Documentation/devicetree/bindings/dma/k3dma.txt +++ b/Documentation/devicetree/bindings/dma/k3dma.txt @@ -3,7 +3,9 @@ See dma.txt first Required properties: -- compatible: Should be "hisilicon,k3-dma-1.0" +- compatible: Must be one of +- "hisilicon,k3-dma-1.0" +- "hisilicon,hisi-pcm-asp-dma-1.0" - reg: Should contain DMA registers location and length. - interrupts: Should contain one interrupt shared by all channel - #dma-cells: see dma.txt, should be 1, para number diff --git a/Documentation/devicetree/bindings/dma/snps-dma.txt b/Documentation/devicetree/bindings/dma/snps-dma.txt index db757df7057df..0bedceed19635 100644 --- a/Documentation/devicetree/bindings/dma/snps-dma.txt +++ b/Documentation/devicetree/bindings/dma/snps-dma.txt @@ -23,8 +23,6 @@ Deprecated properties: Optional properties: -- is_private: The device channels should be marked as private and not for by the - general purpose DMA channel allocator. False if not passed. - multi-block: Multi block transfers supported by hardware. Array property with one cell per channel. 0: not supported, 1 (default): supported. - snps,dma-protection-control: AHB HPROT[3:1] protection setting. diff --git a/Documentation/devicetree/bindings/dma/sprd-dma.txt b/Documentation/devicetree/bindings/dma/sprd-dma.txt index 7a10fea2e51bd..adccea9941f1b 100644 --- a/Documentation/devicetree/bindings/dma/sprd-dma.txt +++ b/Documentation/devicetree/bindings/dma/sprd-dma.txt @@ -31,7 +31,7 @@ DMA clients connected to the Spreadtrum DMA controller must use the format described in the dma.txt file, using a two-cell specifier for each channel. The two cells in order are: 1. A phandle pointing to the DMA controller. -2. The channel id. +2. The slave id. spi0: spi@70a00000{ ... diff --git a/Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt b/Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt index 174af2c45e777..93b6d961dd4fa 100644 --- a/Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt +++ b/Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt @@ -37,10 +37,11 @@ Required properties: Required properties for VDMA: - xlnx,num-fstores: Should be the number of framebuffers as configured in h/w. -Optional properties: -- xlnx,include-sg: Tells configured for Scatter-mode in - the hardware. Optional properties for AXI DMA: +- xlnx,sg-length-width: Should be set to the width in bits of the length + register as configured in h/w. Takes values {8...26}. If the property + is missing or invalid then the default value 23 is used. This is the + maximum value that is supported by all IP versions. - xlnx,mcdma: Tells whether configured for multi-channel mode in the hardware. Optional properties for VDMA: - xlnx,flush-fsync: Tells which channel to Flush on Frame sync. diff --git a/Documentation/devicetree/bindings/net/dsa/dsa.txt b/Documentation/devicetree/bindings/net/dsa/dsa.txt index 35694c0c376b9..d66a5292b9d32 100644 --- a/Documentation/devicetree/bindings/net/dsa/dsa.txt +++ b/Documentation/devicetree/bindings/net/dsa/dsa.txt @@ -71,6 +71,10 @@ properties, described in binding documents: Documentation/devicetree/bindings/net/fixed-link.txt for details. +- local-mac-address : See + Documentation/devicetree/bindings/net/ethernet.txt + for details. + Example The following example shows three switches on three MDIO busses, @@ -97,6 +101,7 @@ linked into one DSA cluster. port@1 { reg = <1>; label = "lan1"; + local-mac-address = [00 00 00 00 00 00]; }; port@2 { diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp-pil.txt b/Documentation/devicetree/bindings/remoteproc/qcom,adsp-pil.txt index a842a782b5576..66af2c30944f4 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,adsp-pil.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,adsp-pil.txt @@ -35,7 +35,7 @@ on the Qualcomm Technology Inc. ADSP Hexagon core. Value type: <stringlist> Definition: List of clock input name strings sorted in the same order as the clocks property. Definition must have - "xo", "sway_cbcr", "lpass_aon", "lpass_ahbs_aon_cbcr", + "xo", "sway_cbcr", "lpass_ahbs_aon_cbcr", "lpass_ahbm_aon_cbcr", "qdsp6ss_xo", "qdsp6ss_sleep" and "qdsp6ss_core". @@ -100,13 +100,12 @@ ADSP, as it is found on SDM845 boards. clocks = <&rpmhcc RPMH_CXO_CLK>, <&gcc GCC_LPASS_SWAY_CLK>, - <&lpasscc LPASS_AUDIO_WRAPPER_AON_CLK>, <&lpasscc LPASS_Q6SS_AHBS_AON_CLK>, <&lpasscc LPASS_Q6SS_AHBM_AON_CLK>, <&lpasscc LPASS_QDSP6SS_XO_CLK>, <&lpasscc LPASS_QDSP6SS_SLEEP_CLK>, <&lpasscc LPASS_QDSP6SS_CORE_CLK>; - clock-names = "xo", "sway_cbcr", "lpass_aon", + clock-names = "xo", "sway_cbcr", "lpass_ahbs_aon_cbcr", "lpass_ahbm_aon_cbcr", "qdsp6ss_xo", "qdsp6ss_sleep", "qdsp6ss_core"; diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt index 9c0cff3a5ed85..292dfda9770d7 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt @@ -19,13 +19,30 @@ on the Qualcomm ADSP Hexagon core. - interrupts-extended: Usage: required Value type: <prop-encoded-array> - Definition: must list the watchdog, fatal IRQs ready, handover and - stop-ack IRQs + Definition: reference to the interrupts that match interrupt-names - interrupt-names: Usage: required Value type: <stringlist> - Definition: must be "wdog", "fatal", "ready", "handover", "stop-ack" + Definition: The interrupts needed depends on the compatible + string: + qcom,msm8974-adsp-pil: + qcom,msm8996-adsp-pil: + qcom,msm8996-slpi-pil: + qcom,qcs404-adsp-pas: + qcom,qcs404-cdsp-pas: + qcom,sdm845-adsp-pas: + qcom,sdm845-cdsp-pas: + must be "wdog", "fatal", "ready", "handover", "stop-ack" + qcom,qcs404-wcss-pas: + must be "wdog", "fatal", "ready", "handover", "stop-ack", + "shutdown-ack" + +- firmware-name: + Usage: optional + Value type: <string> + Definition: must list the relative firmware image path for the + Hexagon Core. - clocks: Usage: required diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt b/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt index 9ff5b0309417c..41ca5df5be5ac 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt @@ -28,24 +28,51 @@ on the Qualcomm Hexagon core. - interrupts-extended: Usage: required Value type: <prop-encoded-array> - Definition: must list the watchdog, fatal IRQs ready, handover and - stop-ack IRQs + Definition: reference to the interrupts that match interrupt-names - interrupt-names: Usage: required Value type: <stringlist> - Definition: must be "wdog", "fatal", "ready", "handover", "stop-ack" + Definition: The interrupts needed depends on the the compatible + string: + qcom,q6v5-pil: + qcom,ipq8074-wcss-pil: + qcom,msm8916-mss-pil: + qcom,msm8974-mss-pil: + must be "wdog", "fatal", "ready", "handover", "stop-ack" + qcom,msm8996-mss-pil: + qcom,sdm845-mss-pil: + must be "wdog", "fatal", "ready", "handover", "stop-ack", + "shutdown-ack" + +- firmware-name: + Usage: optional + Value type: <stringlist> + Definition: must list the relative firmware image paths for mba and + modem. They are used for booting and authenticating the + Hexagon core. - clocks: Usage: required Value type: <phandle> - Definition: reference to the iface, bus and mem clocks to be held on - behalf of the booting of the Hexagon core + Definition: reference to the clocks that match clock-names - clock-names: Usage: required Value type: <stringlist> - Definition: must be "iface", "bus", "mem" + Definition: The clocks needed depend on the compatible string: + qcom,ipq8074-wcss-pil: + no clock names required + qcom,q6v5-pil: + qcom,msm8916-mss-pil: + qcom,msm8974-mss-pil: + must be "iface", "bus", "mem", "xo" + qcom,msm8996-mss-pil: + must be "iface", "bus", "mem", "xo", "gpll0_mss", + "snoc_axi", "mnoc_axi", "pnoc", "qdss" + qcom,sdm845-mss-pil: + must be "iface", "bus", "mem", "xo", "gpll0_mss", + "snoc_axi", "mnoc_axi", "prng" - resets: Usage: required @@ -65,6 +92,19 @@ on the Qualcomm Hexagon core. must be "mss_restart", "pdc_reset" for the modem sub-system on SDM845 SoCs +For the compatible strings below the following supplies are required: + "qcom,q6v5-pil" + "qcom,msm8916-mss-pil", +- cx-supply: +- mx-supply: +- pll-supply: + Usage: required + Value type: <phandle> + Definition: reference to the regulators to be held on behalf of the + booting of the Hexagon core + +For the compatible string below the following supplies are required: + "qcom,msm8974-mss-pil" - cx-supply: - mss-supply: - mx-supply: @@ -74,6 +114,33 @@ on the Qualcomm Hexagon core. Definition: reference to the regulators to be held on behalf of the booting of the Hexagon core +For the compatible string below the following supplies are required: + "qcom,msm8996-mss-pil" +- pll-supply: + Usage: required + Value type: <phandle> + Definition: reference to the regulators to be held on behalf of the + booting of the Hexagon core + +- power-domains: + Usage: required + Value type: <phandle> + Definition: reference to power-domains that match power-domain-names + +- power-domain-names: + Usage: required + Value type: <stringlist> + Definition: The power-domains needed depend on the compatible string: + qcom,q6v5-pil: + qcom,ipq8074-wcss-pil: + qcom,msm8916-mss-pil: + qcom,msm8974-mss-pil: + no power-domain names required + qcom,msm8996-mss-pil: + must be "cx", "mx" + qcom,sdm845-mss-pil: + must be "cx", "mx", "mss", "load_state" + - qcom,smem-states: Usage: required Value type: <phandle> diff --git a/Documentation/driver-api/dmaengine/client.rst b/Documentation/driver-api/dmaengine/client.rst index d728e50105eb3..45953f1715000 100644 --- a/Documentation/driver-api/dmaengine/client.rst +++ b/Documentation/driver-api/dmaengine/client.rst @@ -172,7 +172,7 @@ The details of these operations are: After calling ``dmaengine_submit()`` the submitted transfer descriptor (``struct dma_async_tx_descriptor``) belongs to the DMA engine. - Consequentially, the client must consider invalid the pointer to that + Consequently, the client must consider invalid the pointer to that descriptor. 5. Issue pending DMA requests and wait for callback notification diff --git a/Documentation/driver-api/dmaengine/dmatest.rst b/Documentation/driver-api/dmaengine/dmatest.rst index 8d81f1a7169b7..e78d070bb468f 100644 --- a/Documentation/driver-api/dmaengine/dmatest.rst +++ b/Documentation/driver-api/dmaengine/dmatest.rst @@ -59,6 +59,7 @@ parameter, that specific channel is requested using the dmaengine and a thread is created with the existing parameters. This thread is set as pending and will be executed once run is set to 1. Any parameters set after the thread is created are not applied. + .. hint:: available channel list could be extracted by running the following command:: diff --git a/Documentation/driver-model/devres.txt b/Documentation/driver-model/devres.txt index b277cafce71eb..d7d6f01e81fff 100644 --- a/Documentation/driver-model/devres.txt +++ b/Documentation/driver-model/devres.txt @@ -242,9 +242,11 @@ certainly invest a bit more effort into libata core layer). CLOCK devm_clk_get() + devm_clk_get_optional() devm_clk_put() devm_clk_hw_register() devm_of_clk_add_hw_provider() + devm_clk_hw_register_clkdev() DMA dmaenginem_async_device_register() diff --git a/Documentation/filesystems/cifs/TODO b/Documentation/filesystems/cifs/TODO index 66b3f54aa6dc4..9267f3fb131f9 100644 --- a/Documentation/filesystems/cifs/TODO +++ b/Documentation/filesystems/cifs/TODO @@ -111,7 +111,8 @@ negotiated size) and send larger write sizes to modern servers. 5) Continue to extend the smb3 "buildbot" which does automated xfstesting against Windows, Samba and Azure currently - to add additional tests and -to allow the buildbot to execute the tests faster. +to allow the buildbot to execute the tests faster. The URL for the +buildbot is: http://smb3-test-rhel-75.southcentralus.cloudapp.azure.com 6) Address various coverity warnings (most are not bugs per-se, but the more warnings are addressed, the easier it is to spot real diff --git a/Documentation/filesystems/cifs/cifs.txt b/Documentation/filesystems/cifs/cifs.txt index 67756607246e7..1be3d21c286ec 100644 --- a/Documentation/filesystems/cifs/cifs.txt +++ b/Documentation/filesystems/cifs/cifs.txt @@ -1,16 +1,21 @@ This is the client VFS module for the SMB3 NAS protocol as well - older dialects such as the Common Internet File System (CIFS) + as for older dialects such as the Common Internet File System (CIFS) protocol which was the successor to the Server Message Block (SMB) protocol, the native file sharing mechanism for most early PC operating systems. New and improved versions of CIFS are now - called SMB2 and SMB3. These dialects are also supported by the - CIFS VFS module. CIFS is fully supported by network - file servers such as Windows 2000, 2003, 2008, 2012 and 2016 - as well by Samba (which provides excellent CIFS - server support for Linux and many other operating systems), Apple - systems, as well as most Network Attached Storage vendors, so - this network filesystem client can mount to a wide variety of - servers. + called SMB2 and SMB3. Use of SMB3 (and later, including SMB3.1.1) + is strongly preferred over using older dialects like CIFS due to + security reaasons. All modern dialects, including the most recent, + SMB3.1.1 are supported by the CIFS VFS module. The SMB3 protocol + is implemented and supported by all major file servers + such as all modern versions of Windows (including Windows 2016 + Server), as well as by Samba (which provides excellent + CIFS/SMB2/SMB3 server support and tools for Linux and many other + operating systems). Apple systems also support SMB3 well, as + do most Network Attached Storage vendors, so this network + filesystem client can mount to a wide variety of systems. + It also supports mounting to the cloud (for example + Microsoft Azure), including the necessary security features. The intent of this module is to provide the most advanced network file system function for SMB3 compliant servers, including advanced @@ -24,12 +29,17 @@ cluster file systems for fileserving in some Linux to Linux environments, not just in Linux to Windows (or Linux to Mac) environments. - This filesystem has an mount utility (mount.cifs) that can be obtained from + This filesystem has a mount utility (mount.cifs) and various user space + tools (including smbinfo and setcifsacl) that can be obtained from - https://ftp.samba.org/pub/linux-cifs/cifs-utils/ + https://git.samba.org/?p=cifs-utils.git + or + git://git.samba.org/cifs-utils.git - It must be installed in the directory with the other mount helpers. + mount.cifs should be installed in the directory with the other mount helpers. For more information on the module see the project wiki page at + https://wiki.samba.org/index.php/LinuxCIFS + and https://wiki.samba.org/index.php/LinuxCIFS_utils diff --git a/Documentation/filesystems/f2fs.txt b/Documentation/filesystems/f2fs.txt index e46c2147ddf8e..f7b5e4ff0de3e 100644 --- a/Documentation/filesystems/f2fs.txt +++ b/Documentation/filesystems/f2fs.txt @@ -126,6 +126,8 @@ disable_ext_identify Disable the extension list configured by mkfs, so f2fs does not aware of cold files such as media files. inline_xattr Enable the inline xattrs feature. noinline_xattr Disable the inline xattrs feature. +inline_xattr_size=%u Support configuring inline xattr size, it depends on + flexible inline xattr feature. inline_data Enable the inline data feature: New created small(<~3.4k) files can be written into inode block. inline_dentry Enable the inline dir feature: data in new created diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt index 356156f5c52d2..7de9eee73fcd9 100644 --- a/Documentation/virtual/kvm/api.txt +++ b/Documentation/virtual/kvm/api.txt @@ -45,6 +45,23 @@ the API. The only supported use is one virtual machine per process, and one vcpu per thread. +It is important to note that althought VM ioctls may only be issued from +the process that created the VM, a VM's lifecycle is associated with its +file descriptor, not its creator (process). In other words, the VM and +its resources, *including the associated address space*, are not freed +until the last reference to the VM's file descriptor has been released. +For example, if fork() is issued after ioctl(KVM_CREATE_VM), the VM will +not be freed until both the parent (original) process and its child have +put their references to the VM's file descriptor. + +Because a VM's resources are not freed until the last reference to its +file descriptor is released, creating additional references to a VM via +via fork(), dup(), etc... without careful consideration is strongly +discouraged and may have unwanted side effects, e.g. memory allocated +by and on behalf of the VM's process may not be freed/unaccounted when +the VM is shut down. + + 3. Extensions ------------- diff --git a/Documentation/virtual/kvm/halt-polling.txt b/Documentation/virtual/kvm/halt-polling.txt index 4a84183187697..4f791b128dd27 100644 --- a/Documentation/virtual/kvm/halt-polling.txt +++ b/Documentation/virtual/kvm/halt-polling.txt @@ -53,7 +53,8 @@ the global max polling interval then the polling interval can be increased in the hope that next time during the longer polling interval the wake up source will be received while the host is polling and the latency benefits will be received. The polling interval is grown in the function grow_halt_poll_ns() and -is multiplied by the module parameter halt_poll_ns_grow. +is multiplied by the module parameters halt_poll_ns_grow and +halt_poll_ns_grow_start. In the event that the total block time was greater than the global max polling interval then the host will never poll for long enough (limited by the global @@ -80,22 +81,30 @@ shrunk. These variables are defined in include/linux/kvm_host.h and as module parameters in virt/kvm/kvm_main.c, or arch/powerpc/kvm/book3s_hv.c in the powerpc kvm-hv case. -Module Parameter | Description | Default Value +Module Parameter | Description | Default Value -------------------------------------------------------------------------------- -halt_poll_ns | The global max polling interval | KVM_HALT_POLL_NS_DEFAULT - | which defines the ceiling value | - | of the polling interval for | (per arch value) - | each vcpu. | +halt_poll_ns | The global max polling | KVM_HALT_POLL_NS_DEFAULT + | interval which defines | + | the ceiling value of the | + | polling interval for | (per arch value) + | each vcpu. | -------------------------------------------------------------------------------- -halt_poll_ns_grow | The value by which the halt | 2 - | polling interval is multiplied | - | in the grow_halt_poll_ns() | - | function. | +halt_poll_ns_grow | The value by which the | 2 + | halt polling interval is | + | multiplied in the | + | grow_halt_poll_ns() | + | function. | -------------------------------------------------------------------------------- -halt_poll_ns_shrink | The value by which the halt | 0 - | polling interval is divided in | - | the shrink_halt_poll_ns() | - | function. | +halt_poll_ns_grow_start | The initial value to grow | 10000 + | to from zero in the | + | grow_halt_poll_ns() | + | function. | +-------------------------------------------------------------------------------- +halt_poll_ns_shrink | The value by which the | 0 + | halt polling interval is | + | divided in the | + | shrink_halt_poll_ns() | + | function. | -------------------------------------------------------------------------------- These module parameters can be set from the debugfs files in: diff --git a/Documentation/virtual/kvm/mmu.txt b/Documentation/virtual/kvm/mmu.txt index e507a9e0421ed..f365102c80f5d 100644 --- a/Documentation/virtual/kvm/mmu.txt +++ b/Documentation/virtual/kvm/mmu.txt @@ -224,10 +224,6 @@ Shadow pages contain the following information: A bitmap indicating which sptes in spt point (directly or indirectly) at pages that may be unsynchronized. Used to quickly locate all unsychronized pages reachable from a given page. - mmu_valid_gen: - Generation number of the page. It is compared with kvm->arch.mmu_valid_gen - during hash table lookup, and used to skip invalidated shadow pages (see - "Zapping all pages" below.) clear_spte_count: Only present on 32-bit hosts, where a 64-bit spte cannot be written atomically. The reader uses this while running out of the MMU lock @@ -402,27 +398,6 @@ causes its disallow_lpage to be incremented, thus preventing instantiation of a large spte. The frames at the end of an unaligned memory slot have artificially inflated ->disallow_lpages so they can never be instantiated. -Zapping all pages (page generation count) -========================================= - -For the large memory guests, walking and zapping all pages is really slow -(because there are a lot of pages), and also blocks memory accesses of -all VCPUs because it needs to hold the MMU lock. - -To make it be more scalable, kvm maintains a global generation number -which is stored in kvm->arch.mmu_valid_gen. Every shadow page stores -the current global generation-number into sp->mmu_valid_gen when it -is created. Pages with a mismatching generation number are "obsolete". - -When KVM need zap all shadow pages sptes, it just simply increases the global -generation-number then reload root shadow pages on all vcpus. As the VCPUs -create new shadow page tables, the old pages are not used because of the -mismatching generation number. - -KVM then walks through all pages and zaps obsolete pages. While the zap -operation needs to take the MMU lock, the lock can be released periodically -so that the VCPUs can make progress. - Fast invalidation of MMIO sptes =============================== @@ -435,8 +410,7 @@ shadow pages, and is made more scalable with a similar technique. MMIO sptes have a few spare bits, which are used to store a generation number. The global generation number is stored in kvm_memslots(kvm)->generation, and increased whenever guest memory info -changes. This generation number is distinct from the one described in -the previous section. +changes. When KVM finds an MMIO spte, it checks the generation number of the spte. If the generation number of the spte does not equal the global generation @@ -452,13 +426,16 @@ stored into the MMIO spte. Thus, the MMIO spte might be created based on out-of-date information, but with an up-to-date generation number. To avoid this, the generation number is incremented again after synchronize_srcu -returns; thus, the low bit of kvm_memslots(kvm)->generation is only 1 during a +returns; thus, bit 63 of kvm_memslots(kvm)->generation set to 1 only during a memslot update, while some SRCU readers might be using the old copy. We do not want to use an MMIO sptes created with an odd generation number, and we can do -this without losing a bit in the MMIO spte. The low bit of the generation -is not stored in MMIO spte, and presumed zero when it is extracted out of the -spte. If KVM is unlucky and creates an MMIO spte while the low bit is 1, -the next access to the spte will always be a cache miss. +this without losing a bit in the MMIO spte. The "update in-progress" bit of the +generation is not stored in MMIO spte, and is so is implicitly zero when the +generation is extracted out of the spte. If KVM is unlucky and creates an MMIO +spte while an update is in-progress, the next access to the spte will always be +a cache miss. For example, a subsequent access during the update window will +miss due to the in-progress flag diverging, while an access after the update +window closes will have a higher generation number (as compared to the spte). Further reading diff --git a/MAINTAINERS b/MAINTAINERS index f835946d78ce8..e17ebf70b5480 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3204,6 +3204,7 @@ F: drivers/phy/broadcom/phy-brcm-usb* BROADCOM GENET ETHERNET DRIVER M: Doug Berger <opendmb@gmail.com> M: Florian Fainelli <f.fainelli@gmail.com> +L: bcm-kernel-feedback-list@broadcom.com L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/broadcom/genet/ @@ -3311,6 +3312,7 @@ F: drivers/spi/spi-iproc-qspi.c BROADCOM SYSTEMPORT ETHERNET DRIVER M: Florian Fainelli <f.fainelli@gmail.com> +L: bcm-kernel-feedback-list@broadcom.com L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/broadcom/bcmsysport.* @@ -5276,7 +5278,7 @@ DRM DRIVERS FOR VIVANTE GPU IP M: Lucas Stach <l.stach@pengutronix.de> R: Russell King <linux+etnaviv@armlinux.org.uk> R: Christian Gmeiner <christian.gmeiner@gmail.com> -L: etnaviv@lists.freedesktop.org +L: etnaviv@lists.freedesktop.org (moderated for non-subscribers) L: dri-devel@lists.freedesktop.org S: Maintained F: drivers/gpu/drm/etnaviv/ @@ -8459,6 +8461,7 @@ F: include/linux/kvm* F: include/kvm/iodev.h F: virt/kvm/* F: tools/kvm/ +F: tools/testing/selftests/kvm/ KERNEL VIRTUAL MACHINE FOR AMD-V (KVM/amd) M: Joerg Roedel <joro@8bytes.org> @@ -8468,29 +8471,25 @@ S: Maintained F: arch/x86/include/asm/svm.h F: arch/x86/kvm/svm.c -KERNEL VIRTUAL MACHINE FOR ARM (KVM/arm) +KERNEL VIRTUAL MACHINE FOR ARM/ARM64 (KVM/arm, KVM/arm64) M: Christoffer Dall <christoffer.dall@arm.com> M: Marc Zyngier <marc.zyngier@arm.com> +R: James Morse <james.morse@arm.com> +R: Julien Thierry <julien.thierry@arm.com> +R: Suzuki K Pouloze <suzuki.poulose@arm.com> L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) L: kvmarm@lists.cs.columbia.edu W: http://systems.cs.columbia.edu/projects/kvm-arm T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git -S: Supported +S: Maintained F: arch/arm/include/uapi/asm/kvm* F: arch/arm/include/asm/kvm* F: arch/arm/kvm/ -F: virt/kvm/arm/ -F: include/kvm/arm_* - -KERNEL VIRTUAL MACHINE FOR ARM64 (KVM/arm64) -M: Christoffer Dall <christoffer.dall@arm.com> -M: Marc Zyngier <marc.zyngier@arm.com> -L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) -L: kvmarm@lists.cs.columbia.edu -S: Maintained F: arch/arm64/include/uapi/asm/kvm* F: arch/arm64/include/asm/kvm* F: arch/arm64/kvm/ +F: virt/kvm/arm/ +F: include/kvm/arm_* KERNEL VIRTUAL MACHINE FOR MIPS (KVM/mips) M: James Hogan <jhogan@kernel.org> diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h index 0d0fddb7e7388..976e89b116e5b 100644 --- a/arch/alpha/include/uapi/asm/socket.h +++ b/arch/alpha/include/uapi/asm/socket.h @@ -2,8 +2,8 @@ #ifndef _UAPI_ASM_SOCKET_H #define _UAPI_ASM_SOCKET_H +#include <linux/posix_types.h> #include <asm/sockios.h> -#include <asm/bitsperlong.h> /* For setsockopt(2) */ /* diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 5085a1eab9fc5..054ead960f983 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1310,7 +1310,7 @@ config SCHED_SMT config HAVE_ARM_SCU bool help - This option enables support for the ARM system coherency unit + This option enables support for the ARM snoop control unit config HAVE_ARM_ARCH_TIMER bool "Architected timer support" @@ -1322,7 +1322,6 @@ config HAVE_ARM_ARCH_TIMER config HAVE_ARM_TWD bool - select TIMER_OF if OF help This options enables support for the ARM timer and watchdog unit diff --git a/arch/arm/Kconfig-nommu b/arch/arm/Kconfig-nommu index 1168a03c85255..36c80d3dd93f2 100644 --- a/arch/arm/Kconfig-nommu +++ b/arch/arm/Kconfig-nommu @@ -20,10 +20,12 @@ config DRAM_SIZE config FLASH_MEM_BASE hex 'FLASH Base Address' if SET_MEM_PARAM + depends on CPU_ARM740T || CPU_ARM946E || CPU_ARM940T default 0x00400000 config FLASH_SIZE hex 'FLASH Size' if SET_MEM_PARAM + depends on CPU_ARM740T || CPU_ARM946E || CPU_ARM940T default 0x00400000 config PROCESSOR_ID diff --git a/arch/arm/Makefile b/arch/arm/Makefile index 00000e91ad652..807a7d06c2a08 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -10,7 +10,7 @@ # # Copyright (C) 1995-2001 by Russell King -LDFLAGS_vmlinux :=-p --no-undefined -X --pic-veneer +LDFLAGS_vmlinux := --no-undefined -X --pic-veneer ifeq ($(CONFIG_CPU_ENDIAN_BE8),y) LDFLAGS_vmlinux += --be8 KBUILD_LDFLAGS_MODULE += --be8 diff --git a/arch/arm/boot/bootp/Makefile b/arch/arm/boot/bootp/Makefile index 83e1a076a5d64..981a8d03f064c 100644 --- a/arch/arm/boot/bootp/Makefile +++ b/arch/arm/boot/bootp/Makefile @@ -8,7 +8,7 @@ GCOV_PROFILE := n -LDFLAGS_bootp :=-p --no-undefined -X \ +LDFLAGS_bootp := --no-undefined -X \ --defsym initrd_phys=$(INITRD_PHYS) \ --defsym params_phys=$(PARAMS_PHYS) -T AFLAGS_initrd.o :=-DINITRD=\"$(INITRD)\" diff --git a/arch/arm/boot/bootp/init.S b/arch/arm/boot/bootp/init.S index 78b508075161f..142927e5f485a 100644 --- a/arch/arm/boot/bootp/init.S +++ b/arch/arm/boot/bootp/init.S @@ -44,7 +44,7 @@ _start: add lr, pc, #-0x8 @ lr = current load addr */ movne r10, #0 @ terminator movne r4, #2 @ Size of this entry (2 words) - stmneia r9, {r4, r5, r10} @ Size, ATAG_CORE, terminator + stmiane r9, {r4, r5, r10} @ Size, ATAG_CORE, terminator /* * find the end of the tag list, and then add an INITRD tag on the end. diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index 6114ae6ea4666..9219389bbe612 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -132,8 +132,6 @@ endif ifeq ($(CONFIG_CPU_ENDIAN_BE8),y) LDFLAGS_vmlinux += --be8 endif -# ? -LDFLAGS_vmlinux += -p # Report unresolved symbol references LDFLAGS_vmlinux += --no-undefined # Delete all temporary local symbols diff --git a/arch/arm/boot/compressed/ll_char_wr.S b/arch/arm/boot/compressed/ll_char_wr.S index 8517c8606b4a7..b1dcdb9f4030e 100644 --- a/arch/arm/boot/compressed/ll_char_wr.S +++ b/arch/arm/boot/compressed/ll_char_wr.S @@ -75,7 +75,7 @@ Lrow4bpplp: tst r1, #7 @ avoid using r7 directly after str r7, [r0, -r5]! subne r1, r1, #1 - ldrneb r7, [r6, r1] + ldrbne r7, [r6, r1] bne Lrow4bpplp ldmfd sp!, {r4 - r7, pc} @@ -103,7 +103,7 @@ Lrow8bpplp: sub r0, r0, r5 @ avoid ip stmia r0, {r4, ip} subne r1, r1, #1 - ldrneb r7, [r6, r1] + ldrbne r7, [r6, r1] bne Lrow8bpplp ldmfd sp!, {r4 - r7, pc} diff --git a/arch/arm/boot/dts/imx28-cfa10036.dts b/arch/arm/boot/dts/imx28-cfa10036.dts index d3e3622979c5b..de48b5808ef6e 100644 --- a/arch/arm/boot/dts/imx28-cfa10036.dts +++ b/arch/arm/boot/dts/imx28-cfa10036.dts @@ -11,6 +11,7 @@ /dts-v1/; #include "imx28.dtsi" +#include <dt-bindings/gpio/gpio.h> / { model = "Crystalfontz CFA-10036 Board"; @@ -96,7 +97,7 @@ pinctrl-names = "default"; pinctrl-0 = <&ssd1306_cfa10036>; reg = <0x3c>; - reset-gpios = <&gpio2 7 0>; + reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>; solomon,height = <32>; solomon,width = <128>; solomon,page-offset = <0>; diff --git a/arch/arm/common/mcpm_entry.c b/arch/arm/common/mcpm_entry.c index ad574d20415c2..1b1b82b37ce03 100644 --- a/arch/arm/common/mcpm_entry.c +++ b/arch/arm/common/mcpm_entry.c @@ -381,7 +381,7 @@ static int __init nocache_trampoline(unsigned long _arg) unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); phys_reset_t phys_reset; - mcpm_set_entry_vector(cpu, cluster, cpu_resume); + mcpm_set_entry_vector(cpu, cluster, cpu_resume_no_hyp); setup_mm_for_reboot(); __mcpm_cpu_going_down(cpu, cluster); diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h index f6f485f4744e0..d15b8c99f1b3c 100644 --- a/arch/arm/include/asm/arch_gicv3.h +++ b/arch/arm/include/asm/arch_gicv3.h @@ -55,7 +55,7 @@ #define ICH_VTR __ACCESS_CP15(c12, 4, c11, 1) #define ICH_MISR __ACCESS_CP15(c12, 4, c11, 2) #define ICH_EISR __ACCESS_CP15(c12, 4, c11, 3) -#define ICH_ELSR __ACCESS_CP15(c12, 4, c11, 5) +#define ICH_ELRSR __ACCESS_CP15(c12, 4, c11, 5) #define ICH_VMCR __ACCESS_CP15(c12, 4, c11, 7) #define __LR0(x) __ACCESS_CP15(c12, 4, c12, x) @@ -152,7 +152,7 @@ CPUIF_MAP(ICH_HCR, ICH_HCR_EL2) CPUIF_MAP(ICH_VTR, ICH_VTR_EL2) CPUIF_MAP(ICH_MISR, ICH_MISR_EL2) CPUIF_MAP(ICH_EISR, ICH_EISR_EL2) -CPUIF_MAP(ICH_ELSR, ICH_ELSR_EL2) +CPUIF_MAP(ICH_ELRSR, ICH_ELRSR_EL2) CPUIF_MAP(ICH_VMCR, ICH_VMCR_EL2) CPUIF_MAP(ICH_AP0R3, ICH_AP0R3_EL2) CPUIF_MAP(ICH_AP0R2, ICH_AP0R2_EL2) diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 28a48e0d4cca0..b59921a560da3 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -376,9 +376,9 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .macro usraccoff, instr, reg, ptr, inc, off, cond, abort, t=TUSER() 9999: .if \inc == 1 - \instr\cond\()b\()\t\().w \reg, [\ptr, #\off] + \instr\()b\t\cond\().w \reg, [\ptr, #\off] .elseif \inc == 4 - \instr\cond\()\t\().w \reg, [\ptr, #\off] + \instr\t\cond\().w \reg, [\ptr, #\off] .else .error "Unsupported inc macro argument" .endif @@ -417,9 +417,9 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .rept \rept 9999: .if \inc == 1 - \instr\cond\()b\()\t \reg, [\ptr], #\inc + \instr\()b\t\cond \reg, [\ptr], #\inc .elseif \inc == 4 - \instr\cond\()\t \reg, [\ptr], #\inc + \instr\t\cond \reg, [\ptr], #\inc .else .error "Unsupported inc macro argument" .endif @@ -460,7 +460,7 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .macro check_uaccess, addr:req, size:req, limit:req, tmp:req, bad:req #ifndef CONFIG_CPU_USE_DOMAINS adds \tmp, \addr, #\size - 1 - sbcccs \tmp, \tmp, \limit + sbcscc \tmp, \tmp, \limit bcs \bad #ifdef CONFIG_CPU_SPECTRE movcs \addr, #0 @@ -474,7 +474,7 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) sub \tmp, \limit, #1 subs \tmp, \tmp, \addr @ tmp = limit - 1 - addr addhs \tmp, \tmp, #1 @ if (tmp >= 0) { - subhss \tmp, \tmp, \size @ tmp = limit - (addr + size) } + subshs \tmp, \tmp, \size @ tmp = limit - (addr + size) } movlo \addr, #0 @ if (tmp < 0) addr = NULL csdb #endif diff --git a/arch/arm/include/asm/barrier.h b/arch/arm/include/asm/barrier.h index 69772e742a0ac..83ae97c049d9b 100644 --- a/arch/arm/include/asm/barrier.h +++ b/arch/arm/include/asm/barrier.h @@ -11,6 +11,8 @@ #define sev() __asm__ __volatile__ ("sev" : : : "memory") #define wfe() __asm__ __volatile__ ("wfe" : : : "memory") #define wfi() __asm__ __volatile__ ("wfi" : : : "memory") +#else +#define wfe() do { } while (0) #endif #if __LINUX_ARM_ARCH__ >= 7 diff --git a/arch/arm/include/asm/hardware/entry-macro-iomd.S b/arch/arm/include/asm/hardware/entry-macro-iomd.S index 8c215acd9b573..f7692731e5143 100644 --- a/arch/arm/include/asm/hardware/entry-macro-iomd.S +++ b/arch/arm/include/asm/hardware/entry-macro-iomd.S @@ -16,25 +16,25 @@ ldr \tmp, =irq_prio_h teq \irqstat, #0 #ifdef IOMD_BASE - ldreqb \irqstat, [\base, #IOMD_DMAREQ] @ get dma + ldrbeq \irqstat, [\base, #IOMD_DMAREQ] @ get dma addeq \tmp, \tmp, #256 @ irq_prio_h table size teqeq \irqstat, #0 bne 2406f #endif - ldreqb \irqstat, [\base, #IOMD_IRQREQA] @ get low priority + ldrbeq \irqstat, [\base, #IOMD_IRQREQA] @ get low priority addeq \tmp, \tmp, #256 @ irq_prio_d table size teqeq \irqstat, #0 #ifdef IOMD_IRQREQC - ldreqb \irqstat, [\base, #IOMD_IRQREQC] + ldrbeq \irqstat, [\base, #IOMD_IRQREQC] addeq \tmp, \tmp, #256 @ irq_prio_l table size teqeq \irqstat, #0 #endif #ifdef IOMD_IRQREQD - ldreqb \irqstat, [\base, #IOMD_IRQREQD] + ldrbeq \irqstat, [\base, #IOMD_IRQREQD] addeq \tmp, \tmp, #256 @ irq_prio_lc table size teqeq \irqstat, #0 #endif -2406: ldrneb \irqnr, [\tmp, \irqstat] @ get IRQ number +2406: ldrbne \irqnr, [\tmp, \irqstat] @ get IRQ number .endm /* diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index 77121b713bef3..8927cae7c9666 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -265,6 +265,14 @@ static inline bool kvm_vcpu_dabt_isextabt(struct kvm_vcpu *vcpu) } } +static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) +{ + if (kvm_vcpu_trap_is_iabt(vcpu)) + return false; + + return kvm_vcpu_dabt_iswrite(vcpu); +} + static inline u32 kvm_vcpu_hvc_get_imm(struct kvm_vcpu *vcpu) { return kvm_vcpu_get_hsr(vcpu) & HSR_HVC_IMM_MASK; diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h index 50e89869178a9..770d73257ad93 100644 --- a/arch/arm/include/asm/kvm_host.h +++ b/arch/arm/include/asm/kvm_host.h @@ -26,6 +26,7 @@ #include <asm/kvm_asm.h> #include <asm/kvm_mmio.h> #include <asm/fpstate.h> +#include <asm/smp_plat.h> #include <kvm/arm_arch_timer.h> #define __KVM_HAVE_ARCH_INTC_INITIALIZED @@ -57,10 +58,13 @@ int __attribute_const__ kvm_target_cpu(void); int kvm_reset_vcpu(struct kvm_vcpu *vcpu); void kvm_reset_coprocs(struct kvm_vcpu *vcpu); -struct kvm_arch { - /* VTTBR value associated with below pgd and vmid */ - u64 vttbr; +struct kvm_vmid { + /* The VMID generation used for the virt. memory system */ + u64 vmid_gen; + u32 vmid; +}; +struct kvm_arch { /* The last vcpu id that ran on each physical CPU */ int __percpu *last_vcpu_ran; @@ -70,11 +74,11 @@ struct kvm_arch { */ /* The VMID generation used for the virt. memory system */ - u64 vmid_gen; - u32 vmid; + struct kvm_vmid vmid; /* Stage-2 page table */ pgd_t *pgd; + phys_addr_t pgd_phys; /* Interrupt controller */ struct vgic_dist vgic; @@ -148,6 +152,13 @@ struct kvm_cpu_context { typedef struct kvm_cpu_context kvm_cpu_context_t; +static inline void kvm_init_host_cpu_context(kvm_cpu_context_t *cpu_ctxt, + int cpu) +{ + /* The host's MPIDR is immutable, so let's set it up at boot time */ + cpu_ctxt->cp15[c0_MPIDR] = cpu_logical_map(cpu); +} + struct vcpu_reset_state { unsigned long pc; unsigned long r0; @@ -224,7 +235,35 @@ unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); -unsigned long kvm_call_hyp(void *hypfn, ...); + +unsigned long __kvm_call_hyp(void *hypfn, ...); + +/* + * The has_vhe() part doesn't get emitted, but is used for type-checking. + */ +#define kvm_call_hyp(f, ...) \ + do { \ + if (has_vhe()) { \ + f(__VA_ARGS__); \ + } else { \ + __kvm_call_hyp(kvm_ksym_ref(f), ##__VA_ARGS__); \ + } \ + } while(0) + +#define kvm_call_hyp_ret(f, ...) \ + ({ \ + typeof(f(__VA_ARGS__)) ret; \ + \ + if (has_vhe()) { \ + ret = f(__VA_ARGS__); \ + } else { \ + ret = __kvm_call_hyp(kvm_ksym_ref(f), \ + ##__VA_ARGS__); \ + } \ + \ + ret; \ + }) + void force_vm_exit(const cpumask_t *mask); int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, struct kvm_vcpu_events *events); @@ -275,7 +314,7 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr, * compliant with the PCS!). */ - kvm_call_hyp((void*)hyp_stack_ptr, vector_ptr, pgd_ptr); + __kvm_call_hyp((void*)hyp_stack_ptr, vector_ptr, pgd_ptr); } static inline void __cpu_init_stage2(void) diff --git a/arch/arm/include/asm/kvm_hyp.h b/arch/arm/include/asm/kvm_hyp.h index e93a0cac9addc..87bcd18df8d58 100644 --- a/arch/arm/include/asm/kvm_hyp.h +++ b/arch/arm/include/asm/kvm_hyp.h @@ -40,6 +40,7 @@ #define TTBR1 __ACCESS_CP15_64(1, c2) #define VTTBR __ACCESS_CP15_64(6, c2) #define PAR __ACCESS_CP15_64(0, c7) +#define CNTP_CVAL __ACCESS_CP15_64(2, c14) #define CNTV_CVAL __ACCESS_CP15_64(3, c14) #define CNTVOFF __ACCESS_CP15_64(4, c14) @@ -85,6 +86,7 @@ #define TID_PRIV __ACCESS_CP15(c13, 0, c0, 4) #define HTPIDR __ACCESS_CP15(c13, 4, c0, 2) #define CNTKCTL __ACCESS_CP15(c14, 0, c1, 0) +#define CNTP_CTL __ACCESS_CP15(c14, 0, c2, 1) #define CNTV_CTL __ACCESS_CP15(c14, 0, c3, 1) #define CNTHCTL __ACCESS_CP15(c14, 4, c1, 0) @@ -94,6 +96,8 @@ #define read_sysreg_el0(r) read_sysreg(r##_el0) #define write_sysreg_el0(v, r) write_sysreg(v, r##_el0) +#define cntp_ctl_el0 CNTP_CTL +#define cntp_cval_el0 CNTP_CVAL #define cntv_ctl_el0 CNTV_CTL #define cntv_cval_el0 CNTV_CVAL #define cntvoff_el2 CNTVOFF diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h index 3a875fc1b63ca..2de96a180166e 100644 --- a/arch/arm/include/asm/kvm_mmu.h +++ b/arch/arm/include/asm/kvm_mmu.h @@ -421,9 +421,14 @@ static inline int hyp_map_aux_data(void) static inline void kvm_set_ipa_limit(void) {} -static inline bool kvm_cpu_has_cnp(void) +static __always_inline u64 kvm_get_vttbr(struct kvm *kvm) { - return false; + struct kvm_vmid *vmid = &kvm->arch.vmid; + u64 vmid_field, baddr; + + baddr = kvm->arch.pgd_phys; + vmid_field = (u64)vmid->vmid << VTTBR_VMID_SHIFT; + return kvm_phys_to_vttbr(baddr) | vmid_field; } #endif /* !__ASSEMBLY__ */ diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index a757401129f95..48ce1b19069b6 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -125,6 +125,9 @@ extern pgprot_t pgprot_s2_device; #define pgprot_stronglyordered(prot) \ __pgprot_modify(prot, L_PTE_MT_MASK, L_PTE_MT_UNCACHED) +#define pgprot_device(prot) \ + __pgprot_modify(prot, L_PTE_MT_MASK, L_PTE_MT_DEV_SHARED | L_PTE_SHARED | L_PTE_DIRTY | L_PTE_XN) + #ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE #define pgprot_dmacoherent(prot) \ __pgprot_modify(prot, L_PTE_MT_MASK, L_PTE_MT_BUFFERABLE | L_PTE_XN) diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h index 120f4c9bbfde2..57fe73ea0f725 100644 --- a/arch/arm/include/asm/processor.h +++ b/arch/arm/include/asm/processor.h @@ -89,7 +89,11 @@ extern void release_thread(struct task_struct *); unsigned long get_wchan(struct task_struct *p); #if __LINUX_ARM_ARCH__ == 6 || defined(CONFIG_ARM_ERRATA_754327) -#define cpu_relax() smp_mb() +#define cpu_relax() \ + do { \ + smp_mb(); \ + __asm__ __volatile__("nop; nop; nop; nop; nop; nop; nop; nop; nop; nop;"); \ + } while (0) #else #define cpu_relax() barrier() #endif diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index 709a55989cb06..451ae684aaf48 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -67,7 +67,6 @@ struct secondary_data { void *stack; }; extern struct secondary_data secondary_data; -extern volatile int pen_release; extern void secondary_startup(void); extern void secondary_startup_arm(void); diff --git a/arch/arm/include/asm/smp_twd.h b/arch/arm/include/asm/smp_twd.h index 312784ee9936a..c729d2113a245 100644 --- a/arch/arm/include/asm/smp_twd.h +++ b/arch/arm/include/asm/smp_twd.h @@ -19,20 +19,4 @@ #define TWD_TIMER_CONTROL_PERIODIC (1 << 1) #define TWD_TIMER_CONTROL_IT_ENABLE (1 << 2) -#include <linux/ioport.h> - -struct twd_local_timer { - struct resource res[2]; -}; - -#define DEFINE_TWD_LOCAL_TIMER(name,base,irq) \ -struct twd_local_timer name __initdata = { \ - .res = { \ - DEFINE_RES_MEM(base, 0x10), \ - DEFINE_RES_IRQ(irq), \ - }, \ -}; - -int twd_local_timer_register(struct twd_local_timer *); - #endif diff --git a/arch/arm/include/asm/spinlock.h b/arch/arm/include/asm/spinlock.h index 099c78fcf62d4..8f009e788ad40 100644 --- a/arch/arm/include/asm/spinlock.h +++ b/arch/arm/include/asm/spinlock.h @@ -210,11 +210,12 @@ static inline void arch_read_lock(arch_rwlock_t *rw) prefetchw(&rw->lock); __asm__ __volatile__( +" .syntax unified\n" "1: ldrex %0, [%2]\n" " adds %0, %0, #1\n" " strexpl %1, %0, [%2]\n" WFE("mi") -" rsbpls %0, %1, #0\n" +" rsbspl %0, %1, #0\n" " bmi 1b" : "=&r" (tmp), "=&r" (tmp2) : "r" (&rw->lock) diff --git a/arch/arm/include/asm/suspend.h b/arch/arm/include/asm/suspend.h index 452bbdcbcc835..506314265c6f1 100644 --- a/arch/arm/include/asm/suspend.h +++ b/arch/arm/include/asm/suspend.h @@ -10,6 +10,7 @@ struct sleep_save_sp { }; extern void cpu_resume(void); +extern void cpu_resume_no_hyp(void); extern void cpu_resume_arm(void); extern int cpu_suspend(unsigned long, int (*)(unsigned long)); diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index ae5a0df5316e5..dff49845eb876 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -85,7 +85,8 @@ static inline void set_fs(mm_segment_t fs) #define __range_ok(addr, size) ({ \ unsigned long flag, roksum; \ __chk_user_ptr(addr); \ - __asm__("adds %1, %2, %3; sbcccs %1, %1, %0; movcc %0, #0" \ + __asm__(".syntax unified\n" \ + "adds %1, %2, %3; sbcscc %1, %1, %0; movcc %0, #0" \ : "=&r" (flag), "=&r" (roksum) \ : "r" (addr), "Ir" (size), "0" (current_thread_info()->addr_limit) \ : "cc"); \ diff --git a/arch/arm/include/asm/v7m.h b/arch/arm/include/asm/v7m.h index 187ccf6496ad6..2cb00d15831b9 100644 --- a/arch/arm/include/asm/v7m.h +++ b/arch/arm/include/asm/v7m.h @@ -49,7 +49,7 @@ * (0 -> msp; 1 -> psp). Bits [1:0] are fixed to 0b01. */ #define EXC_RET_STACK_MASK 0x00000004 -#define EXC_RET_THREADMODE_PROCESSSTACK 0xfffffffd +#define EXC_RET_THREADMODE_PROCESSSTACK (3 << 2) /* Cache related definitions */ diff --git a/arch/arm/include/asm/vfpmacros.h b/arch/arm/include/asm/vfpmacros.h index ef5dfedacd8d6..628c336e8e3b2 100644 --- a/arch/arm/include/asm/vfpmacros.h +++ b/arch/arm/include/asm/vfpmacros.h @@ -29,13 +29,13 @@ ldr \tmp, =elf_hwcap @ may not have MVFR regs ldr \tmp, [\tmp, #0] tst \tmp, #HWCAP_VFPD32 - ldcnel p11, cr0, [\base],#32*4 @ FLDMIAD \base!, {d16-d31} + ldclne p11, cr0, [\base],#32*4 @ FLDMIAD \base!, {d16-d31} addeq \base, \base, #32*4 @ step over unused register space #else VFPFMRX \tmp, MVFR0 @ Media and VFP Feature Register 0 and \tmp, \tmp, #MVFR0_A_SIMD_MASK @ A_SIMD field cmp \tmp, #2 @ 32 x 64bit registers? - ldceql p11, cr0, [\base],#32*4 @ FLDMIAD \base!, {d16-d31} + ldcleq p11, cr0, [\base],#32*4 @ FLDMIAD \base!, {d16-d31} addne \base, \base, #32*4 @ step over unused register space #endif #endif @@ -53,13 +53,13 @@ ldr \tmp, =elf_hwcap @ may not have MVFR regs ldr \tmp, [\tmp, #0] tst \tmp, #HWCAP_VFPD32 - stcnel p11, cr0, [\base],#32*4 @ FSTMIAD \base!, {d16-d31} + stclne p11, cr0, [\base],#32*4 @ FSTMIAD \base!, {d16-d31} addeq \base, \base, #32*4 @ step over unused register space #else VFPFMRX \tmp, MVFR0 @ Media and VFP Feature Register 0 and \tmp, \tmp, #MVFR0_A_SIMD_MASK @ A_SIMD field cmp \tmp, #2 @ 32 x 64bit registers? - stceql p11, cr0, [\base],#32*4 @ FSTMIAD \base!, {d16-d31} + stcleq p11, cr0, [\base],#32*4 @ FSTMIAD \base!, {d16-d31} addne \base, \base, #32*4 @ step over unused register space #endif #endif diff --git a/arch/arm/include/debug/tegra.S b/arch/arm/include/debug/tegra.S index 3bc80599c0225..4a5a645c76e2f 100644 --- a/arch/arm/include/debug/tegra.S +++ b/arch/arm/include/debug/tegra.S @@ -173,7 +173,7 @@ .macro senduart, rd, rx cmp \rx, #0 - strneb \rd, [\rx, #UART_TX << UART_SHIFT] + strbne \rd, [\rx, #UART_TX << UART_SHIFT] 1001: .endm diff --git a/arch/arm/kernel/debug.S b/arch/arm/kernel/debug.S index b795dc2408c05..b9f94e03d916a 100644 --- a/arch/arm/kernel/debug.S +++ b/arch/arm/kernel/debug.S @@ -86,7 +86,7 @@ hexbuf_rel: .long hexbuf_addr - . ENTRY(printascii) addruart_current r3, r1, r2 1: teq r0, #0 - ldrneb r1, [r0], #1 + ldrbne r1, [r0], #1 teqne r1, #0 reteq lr 2: teq r1, #'\n' diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index e85a3af9ddeb5..ce4aea57130af 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -636,7 +636,7 @@ call_fpe: @ Test if we need to give access to iWMMXt coprocessors ldr r5, [r10, #TI_FLAGS] rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only - movcss r7, r5, lsr #(TIF_USING_IWMMXT + 1) + movscs r7, r5, lsr #(TIF_USING_IWMMXT + 1) bcs iwmmxt_task_enable #endif ARM( add pc, pc, r8, lsr #6 ) @@ -872,7 +872,7 @@ __kuser_cmpxchg64: @ 0xffff0f60 smp_dmb arm 1: ldrexd r0, r1, [r2] @ load current val eors r3, r0, r4 @ compare with oldval (1) - eoreqs r3, r1, r5 @ compare with oldval (2) + eorseq r3, r1, r5 @ compare with oldval (2) strexdeq r3, r6, r7, [r2] @ store newval if eq teqeq r3, #1 @ success? beq 1b @ if no then retry @@ -896,8 +896,8 @@ __kuser_cmpxchg64: @ 0xffff0f60 ldmia r1, {r6, lr} @ load new val 1: ldmia r2, {r0, r1} @ load current val eors r3, r0, r4 @ compare with oldval (1) - eoreqs r3, r1, r5 @ compare with oldval (2) -2: stmeqia r2, {r6, lr} @ store newval if eq + eorseq r3, r1, r5 @ compare with oldval (2) +2: stmiaeq r2, {r6, lr} @ store newval if eq rsbs r0, r3, #0 @ set return val and C flag ldmfd sp!, {r4, r5, r6, pc} @@ -911,7 +911,7 @@ kuser_cmpxchg64_fixup: mov r7, #0xffff0fff sub r7, r7, #(0xffff0fff - (0xffff0f60 + (1b - __kuser_cmpxchg64))) subs r8, r4, r7 - rsbcss r8, r8, #(2b - 1b) + rsbscs r8, r8, #(2b - 1b) strcs r7, [sp, #S_PC] #if __LINUX_ARM_ARCH__ < 6 bcc kuser_cmpxchg32_fixup @@ -969,7 +969,7 @@ kuser_cmpxchg32_fixup: mov r7, #0xffff0fff sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg))) subs r8, r4, r7 - rsbcss r8, r8, #(2b - 1b) + rsbscs r8, r8, #(2b - 1b) strcs r7, [sp, #S_PC] ret lr .previous diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 0465d65d23de5..f7649adef505e 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -373,7 +373,7 @@ sys_syscall: movhs scno, #0 csdb #endif - stmloia sp, {r5, r6} @ shuffle args + stmialo sp, {r5, r6} @ shuffle args movlo r0, r1 movlo r1, r2 movlo r2, r3 diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 773424843d6ef..32051ec5b33fa 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -127,7 +127,8 @@ */ .macro v7m_exception_slow_exit ret_r0 cpsid i - ldr lr, =EXC_RET_THREADMODE_PROCESSSTACK + ldr lr, =exc_ret + ldr lr, [lr] @ read original r12, sp, lr, pc and xPSR add r12, sp, #S_IP @@ -387,8 +388,8 @@ badr lr, \ret @ return address .if \reload add r1, sp, #S_R0 + S_OFF @ pointer to regs - ldmccia r1, {r0 - r6} @ reload r0-r6 - stmccia sp, {r4, r5} @ update stack arguments + ldmiacc r1, {r0 - r6} @ reload r0-r6 + stmiacc sp, {r4, r5} @ update stack arguments .endif ldrcc pc, [\table, \tmp, lsl #2] @ call sys_* routine #else @@ -396,8 +397,8 @@ badr lr, \ret @ return address .if \reload add r1, sp, #S_R0 + S_OFF @ pointer to regs - ldmccia r1, {r0 - r6} @ reload r0-r6 - stmccia sp, {r4, r5} @ update stack arguments + ldmiacc r1, {r0 - r6} @ reload r0-r6 + stmiacc sp, {r4, r5} @ update stack arguments .endif ldrcc pc, [\table, \nr, lsl #2] @ call sys_* routine #endif diff --git a/arch/arm/kernel/entry-v7m.S b/arch/arm/kernel/entry-v7m.S index abcf478485259..19d2dcd6530dc 100644 --- a/arch/arm/kernel/entry-v7m.S +++ b/arch/arm/kernel/entry-v7m.S @@ -146,3 +146,7 @@ ENTRY(vector_table) .rept CONFIG_CPU_V7M_NUM_IRQ .long __irq_entry @ External Interrupts .endr + .align 2 + .globl exc_ret +exc_ret: + .space 4 diff --git a/arch/arm/kernel/head-nommu.S b/arch/arm/kernel/head-nommu.S index ec29de2500764..c08d2d890f7b9 100644 --- a/arch/arm/kernel/head-nommu.S +++ b/arch/arm/kernel/head-nommu.S @@ -439,8 +439,8 @@ M_CLASS(str r6, [r12, #PMSAv8_RLAR_A(3)]) str r5, [r12, #PMSAv8_RBAR_A(0)] str r6, [r12, #PMSAv8_RLAR_A(0)] #else - mcr p15, 0, r5, c6, c10, 1 @ PRBAR4 - mcr p15, 0, r6, c6, c10, 2 @ PRLAR4 + mcr p15, 0, r5, c6, c10, 0 @ PRBAR4 + mcr p15, 0, r6, c6, c10, 1 @ PRLAR4 #endif #endif ret lr diff --git a/arch/arm/kernel/hyp-stub.S b/arch/arm/kernel/hyp-stub.S index 60146e32619a5..82a942894fc04 100644 --- a/arch/arm/kernel/hyp-stub.S +++ b/arch/arm/kernel/hyp-stub.S @@ -180,8 +180,8 @@ ARM_BE8(orr r7, r7, #(1 << 25)) @ HSCTLR.EE @ Check whether GICv3 system registers are available mrc p15, 0, r7, c0, c1, 1 @ ID_PFR1 ubfx r7, r7, #28, #4 - cmp r7, #1 - bne 2f + teq r7, #0 + beq 2f @ Enable system register accesses mrc p15, 4, r7, c12, c9, 5 @ ICC_HSRE diff --git a/arch/arm/kernel/machine_kexec.c b/arch/arm/kernel/machine_kexec.c index dd2eb5f76b9f0..76300f3813e89 100644 --- a/arch/arm/kernel/machine_kexec.c +++ b/arch/arm/kernel/machine_kexec.c @@ -91,8 +91,11 @@ void machine_crash_nonpanic_core(void *unused) set_cpu_online(smp_processor_id(), false); atomic_dec(&waiting_for_crash_ipi); - while (1) + + while (1) { cpu_relax(); + wfe(); + } } void crash_smp_send_stop(void) diff --git a/arch/arm/kernel/patch.c b/arch/arm/kernel/patch.c index a50dc00d79a27..d0a05a3bdb965 100644 --- a/arch/arm/kernel/patch.c +++ b/arch/arm/kernel/patch.c @@ -16,7 +16,7 @@ struct patch { unsigned int insn; }; -static DEFINE_SPINLOCK(patch_lock); +static DEFINE_RAW_SPINLOCK(patch_lock); static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags) __acquires(&patch_lock) @@ -33,7 +33,7 @@ static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags) return addr; if (flags) - spin_lock_irqsave(&patch_lock, *flags); + raw_spin_lock_irqsave(&patch_lock, *flags); else __acquire(&patch_lock); @@ -48,7 +48,7 @@ static void __kprobes patch_unmap(int fixmap, unsigned long *flags) clear_fixmap(fixmap); if (flags) - spin_unlock_irqrestore(&patch_lock, *flags); + raw_spin_unlock_irqrestore(&patch_lock, *flags); else __release(&patch_lock); } diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S index a8257fc9cf2a9..5dc8b80bb6938 100644 --- a/arch/arm/kernel/sleep.S +++ b/arch/arm/kernel/sleep.S @@ -120,6 +120,14 @@ ENDPROC(cpu_resume_after_mmu) .text .align +#ifdef CONFIG_MCPM + .arm +THUMB( .thumb ) +ENTRY(cpu_resume_no_hyp) +ARM_BE8(setend be) @ ensure we are in BE mode + b no_hyp +#endif + #ifdef CONFIG_MMU .arm ENTRY(cpu_resume_arm) @@ -135,6 +143,7 @@ ARM_BE8(setend be) @ ensure we are in BE mode bl __hyp_stub_install_secondary #endif safe_svcmode_maskall r1 +no_hyp: mov r1, #0 ALT_SMP(mrc p15, 0, r0, c0, c0, 5) ALT_UP_B(1f) @@ -164,6 +173,9 @@ ENDPROC(cpu_resume) #ifdef CONFIG_MMU ENDPROC(cpu_resume_arm) #endif +#ifdef CONFIG_MCPM +ENDPROC(cpu_resume_no_hyp) +#endif .align 2 _sleep_save_sp: diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 1d6f5ea522f49..facd4240ca02c 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -62,12 +62,6 @@ */ struct secondary_data secondary_data; -/* - * control for which core is the next to come out of the secondary - * boot "holding pen" - */ -volatile int pen_release = -1; - enum ipi_msg_type { IPI_WAKEUP, IPI_TIMER, @@ -604,8 +598,10 @@ static void ipi_cpu_stop(unsigned int cpu) local_fiq_disable(); local_irq_disable(); - while (1) + while (1) { cpu_relax(); + wfe(); + } } static DEFINE_PER_CPU(struct completion *, cpu_completion); diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c index b30eafeef0963..3cdc399b9fc32 100644 --- a/arch/arm/kernel/smp_twd.c +++ b/arch/arm/kernel/smp_twd.c @@ -100,8 +100,6 @@ static void twd_timer_stop(void) disable_percpu_irq(clk->irq); } -#ifdef CONFIG_COMMON_CLK - /* * Updates clockevent frequency when the cpu frequency changes. * Called on the cpu that is changing frequency with interrupts disabled. @@ -143,54 +141,6 @@ static int twd_clk_init(void) } core_initcall(twd_clk_init); -#elif defined (CONFIG_CPU_FREQ) - -#include <linux/cpufreq.h> - -/* - * Updates clockevent frequency when the cpu frequency changes. - * Called on the cpu that is changing frequency with interrupts disabled. - */ -static void twd_update_frequency(void *data) -{ - twd_timer_rate = clk_get_rate(twd_clk); - - clockevents_update_freq(raw_cpu_ptr(twd_evt), twd_timer_rate); -} - -static int twd_cpufreq_transition(struct notifier_block *nb, - unsigned long state, void *data) -{ - struct cpufreq_freqs *freqs = data; - - /* - * The twd clock events must be reprogrammed to account for the new - * frequency. The timer is local to a cpu, so cross-call to the - * changing cpu. - */ - if (state == CPUFREQ_POSTCHANGE) - smp_call_function_single(freqs->cpu, twd_update_frequency, - NULL, 1); - - return NOTIFY_OK; -} - -static struct notifier_block twd_cpufreq_nb = { - .notifier_call = twd_cpufreq_transition, -}; - -static int twd_cpufreq_init(void) -{ - if (twd_evt && raw_cpu_ptr(twd_evt) && !IS_ERR(twd_clk)) - return cpufreq_register_notifier(&twd_cpufreq_nb, - CPUFREQ_TRANSITION_NOTIFIER); - - return 0; -} -core_initcall(twd_cpufreq_init); - -#endif - static void twd_calibrate_rate(void) { unsigned long count; @@ -366,21 +316,6 @@ out_free: return err; } -int __init twd_local_timer_register(struct twd_local_timer *tlt) -{ - if (twd_base || twd_evt) - return -EBUSY; - - twd_ppi = tlt->res[1].start; - - twd_base = ioremap(tlt->res[0].start, resource_size(&tlt->res[0])); - if (!twd_base) - return -ENOMEM; - - return twd_local_timer_common_register(NULL); -} - -#ifdef CONFIG_OF static int __init twd_local_timer_of_register(struct device_node *np) { int err; @@ -406,4 +341,3 @@ out: TIMER_OF_DECLARE(arm_twd_a9, "arm,cortex-a9-twd-timer", twd_local_timer_of_register); TIMER_OF_DECLARE(arm_twd_a5, "arm,cortex-a5-twd-timer", twd_local_timer_of_register); TIMER_OF_DECLARE(arm_twd_11mp, "arm,arm11mp-twd-timer", twd_local_timer_of_register); -#endif diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index 0bee233fef9a3..314cfb232a635 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -93,7 +93,7 @@ extern const struct unwind_idx __start_unwind_idx[]; static const struct unwind_idx *__origin_unwind_idx; extern const struct unwind_idx __stop_unwind_idx[]; -static DEFINE_SPINLOCK(unwind_lock); +static DEFINE_RAW_SPINLOCK(unwind_lock); static LIST_HEAD(unwind_tables); /* Convert a prel31 symbol to an absolute address */ @@ -201,7 +201,7 @@ static const struct unwind_idx *unwind_find_idx(unsigned long addr) /* module unwind tables */ struct unwind_table *table; - spin_lock_irqsave(&unwind_lock, flags); + raw_spin_lock_irqsave(&unwind_lock, flags); list_for_each_entry(table, &unwind_tables, list) { if (addr >= table->begin_addr && addr < table->end_addr) { @@ -213,7 +213,7 @@ static const struct unwind_idx *unwind_find_idx(unsigned long addr) break; } } - spin_unlock_irqrestore(&unwind_lock, flags); + raw_spin_unlock_irqrestore(&unwind_lock, flags); } pr_debug("%s: idx = %p\n", __func__, idx); @@ -529,9 +529,9 @@ struct unwind_table *unwind_table_add(unsigned long start, unsigned long size, tab->begin_addr = text_addr; tab->end_addr = text_addr + text_size; - spin_lock_irqsave(&unwind_lock, flags); + raw_spin_lock_irqsave(&unwind_lock, flags); list_add_tail(&tab->list, &unwind_tables); - spin_unlock_irqrestore(&unwind_lock, flags); + raw_spin_unlock_irqrestore(&unwind_lock, flags); return tab; } @@ -543,9 +543,9 @@ void unwind_table_del(struct unwind_table *tab) if (!tab) return; - spin_lock_irqsave(&unwind_lock, flags); + raw_spin_lock_irqsave(&unwind_lock, flags); list_del(&tab->list); - spin_unlock_irqrestore(&unwind_lock, flags); + raw_spin_unlock_irqrestore(&unwind_lock, flags); kfree(tab); } diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile index 48de846f22464..531e59f5be9c8 100644 --- a/arch/arm/kvm/Makefile +++ b/arch/arm/kvm/Makefile @@ -8,9 +8,8 @@ ifeq ($(plus_virt),+virt) plus_virt_def := -DREQUIRES_VIRT=1 endif -ccflags-y += -Iarch/arm/kvm -Ivirt/kvm/arm/vgic -CFLAGS_arm.o := -I. $(plus_virt_def) -CFLAGS_mmu.o := -I. +ccflags-y += -I $(srctree)/$(src) -I $(srctree)/virt/kvm/arm/vgic +CFLAGS_arm.o := $(plus_virt_def) AFLAGS_init.o := -Wa,-march=armv7-a$(plus_virt) AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt) diff --git a/arch/arm/kvm/coproc.c b/arch/arm/kvm/coproc.c index e8bd288fd5be9..14915c78bd99b 100644 --- a/arch/arm/kvm/coproc.c +++ b/arch/arm/kvm/coproc.c @@ -293,15 +293,16 @@ static bool access_cntp_tval(struct kvm_vcpu *vcpu, const struct coproc_params *p, const struct coproc_reg *r) { - u64 now = kvm_phys_timer_read(); - u64 val; + u32 val; if (p->is_write) { val = *vcpu_reg(vcpu, p->Rt1); - kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL, val + now); + kvm_arm_timer_write_sysreg(vcpu, + TIMER_PTIMER, TIMER_REG_TVAL, val); } else { - val = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL); - *vcpu_reg(vcpu, p->Rt1) = val - now; + val = kvm_arm_timer_read_sysreg(vcpu, + TIMER_PTIMER, TIMER_REG_TVAL); + *vcpu_reg(vcpu, p->Rt1) = val; } return true; @@ -315,9 +316,11 @@ static bool access_cntp_ctl(struct kvm_vcpu *vcpu, if (p->is_write) { val = *vcpu_reg(vcpu, p->Rt1); - kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CTL, val); + kvm_arm_timer_write_sysreg(vcpu, + TIMER_PTIMER, TIMER_REG_CTL, val); } else { - val = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CTL); + val = kvm_arm_timer_read_sysreg(vcpu, + TIMER_PTIMER, TIMER_REG_CTL); *vcpu_reg(vcpu, p->Rt1) = val; } @@ -333,9 +336,11 @@ static bool access_cntp_cval(struct kvm_vcpu *vcpu, if (p->is_write) { val = (u64)*vcpu_reg(vcpu, p->Rt2) << 32; val |= *vcpu_reg(vcpu, p->Rt1); - kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL, val); + kvm_arm_timer_write_sysreg(vcpu, + TIMER_PTIMER, TIMER_REG_CVAL, val); } else { - val = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL); + val = kvm_arm_timer_read_sysreg(vcpu, + TIMER_PTIMER, TIMER_REG_CVAL); *vcpu_reg(vcpu, p->Rt1) = val; *vcpu_reg(vcpu, p->Rt2) = val >> 32; } diff --git a/arch/arm/kvm/hyp/cp15-sr.c b/arch/arm/kvm/hyp/cp15-sr.c index c4782812714cf..8bf895ec6e042 100644 --- a/arch/arm/kvm/hyp/cp15-sr.c +++ b/arch/arm/kvm/hyp/cp15-sr.c @@ -27,7 +27,6 @@ static u64 *cp15_64(struct kvm_cpu_context *ctxt, int idx) void __hyp_text __sysreg_save_state(struct kvm_cpu_context *ctxt) { - ctxt->cp15[c0_MPIDR] = read_sysreg(VMPIDR); ctxt->cp15[c0_CSSELR] = read_sysreg(CSSELR); ctxt->cp15[c1_SCTLR] = read_sysreg(SCTLR); ctxt->cp15[c1_CPACR] = read_sysreg(CPACR); diff --git a/arch/arm/kvm/hyp/hyp-entry.S b/arch/arm/kvm/hyp/hyp-entry.S index aa3f9a9837aca..6ed3cf23fe890 100644 --- a/arch/arm/kvm/hyp/hyp-entry.S +++ b/arch/arm/kvm/hyp/hyp-entry.S @@ -176,7 +176,7 @@ THUMB( orr lr, lr, #PSR_T_BIT ) msr spsr_cxsf, lr ldr lr, =panic msr ELR_hyp, lr - ldr lr, =kvm_call_hyp + ldr lr, =__kvm_call_hyp clrex eret ENDPROC(__hyp_do_panic) diff --git a/arch/arm/kvm/hyp/switch.c b/arch/arm/kvm/hyp/switch.c index acf1c37fa49c2..3b058a5d7c5f1 100644 --- a/arch/arm/kvm/hyp/switch.c +++ b/arch/arm/kvm/hyp/switch.c @@ -77,7 +77,7 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu) static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu) { struct kvm *kvm = kern_hyp_va(vcpu->kvm); - write_sysreg(kvm->arch.vttbr, VTTBR); + write_sysreg(kvm_get_vttbr(kvm), VTTBR); write_sysreg(vcpu->arch.midr, VPIDR); } diff --git a/arch/arm/kvm/hyp/tlb.c b/arch/arm/kvm/hyp/tlb.c index c0edd450e1045..8e4afba736357 100644 --- a/arch/arm/kvm/hyp/tlb.c +++ b/arch/arm/kvm/hyp/tlb.c @@ -41,7 +41,7 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm) /* Switch to requested VMID */ kvm = kern_hyp_va(kvm); - write_sysreg(kvm->arch.vttbr, VTTBR); + write_sysreg(kvm_get_vttbr(kvm), VTTBR); isb(); write_sysreg(0, TLBIALLIS); @@ -61,7 +61,7 @@ void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu) struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm); /* Switch to requested VMID */ - write_sysreg(kvm->arch.vttbr, VTTBR); + write_sysreg(kvm_get_vttbr(kvm), VTTBR); isb(); write_sysreg(0, TLBIALL); diff --git a/arch/arm/kvm/interrupts.S b/arch/arm/kvm/interrupts.S index 80a1d6cd261ce..a08e6419ebe90 100644 --- a/arch/arm/kvm/interrupts.S +++ b/arch/arm/kvm/interrupts.S @@ -42,7 +42,7 @@ * r12: caller save * rest: callee save */ -ENTRY(kvm_call_hyp) +ENTRY(__kvm_call_hyp) hvc #0 bx lr -ENDPROC(kvm_call_hyp) +ENDPROC(__kvm_call_hyp) diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile index ad25fd1872c7d..0bff0176db2c4 100644 --- a/arch/arm/lib/Makefile +++ b/arch/arm/lib/Makefile @@ -39,7 +39,7 @@ $(obj)/csumpartialcopy.o: $(obj)/csumpartialcopygeneric.S $(obj)/csumpartialcopyuser.o: $(obj)/csumpartialcopygeneric.S ifeq ($(CONFIG_KERNEL_MODE_NEON),y) - NEON_FLAGS := -mfloat-abi=softfp -mfpu=neon + NEON_FLAGS := -march=armv7-a -mfloat-abi=softfp -mfpu=neon CFLAGS_xor-neon.o += $(NEON_FLAGS) obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o endif diff --git a/arch/arm/lib/bitops.h b/arch/arm/lib/bitops.h index 93cddab73072c..95bd359912889 100644 --- a/arch/arm/lib/bitops.h +++ b/arch/arm/lib/bitops.h @@ -7,7 +7,7 @@ ENTRY( \name ) UNWIND( .fnstart ) ands ip, r1, #3 - strneb r1, [ip] @ assert word-aligned + strbne r1, [ip] @ assert word-aligned mov r2, #1 and r3, r0, #31 @ Get bit offset mov r0, r0, lsr #5 @@ -32,7 +32,7 @@ ENDPROC(\name ) ENTRY( \name ) UNWIND( .fnstart ) ands ip, r1, #3 - strneb r1, [ip] @ assert word-aligned + strbne r1, [ip] @ assert word-aligned mov r2, #1 and r3, r0, #31 @ Get bit offset mov r0, r0, lsr #5 @@ -62,7 +62,7 @@ ENDPROC(\name ) ENTRY( \name ) UNWIND( .fnstart ) ands ip, r1, #3 - strneb r1, [ip] @ assert word-aligned + strbne r1, [ip] @ assert word-aligned and r2, r0, #31 mov r0, r0, lsr #5 mov r3, #1 @@ -89,7 +89,7 @@ ENDPROC(\name ) ENTRY( \name ) UNWIND( .fnstart ) ands ip, r1, #3 - strneb r1, [ip] @ assert word-aligned + strbne r1, [ip] @ assert word-aligned and r3, r0, #31 mov r0, r0, lsr #5 save_and_disable_irqs ip diff --git a/arch/arm/lib/clear_user.S b/arch/arm/lib/clear_user.S index e936352ccb001..55946e3fa2ba8 100644 --- a/arch/arm/lib/clear_user.S +++ b/arch/arm/lib/clear_user.S @@ -44,7 +44,7 @@ UNWIND(.save {r1, lr}) strusr r2, r0, 1, ne, rept=2 tst r1, #1 @ x1 x0 x1 x0 x1 x0 x1 it ne @ explicit IT needed for the label -USER( strnebt r2, [r0]) +USER( strbtne r2, [r0]) mov r0, #0 ldmfd sp!, {r1, pc} UNWIND(.fnend) diff --git a/arch/arm/lib/copy_from_user.S b/arch/arm/lib/copy_from_user.S index 0d4c189c7f4f0..6a3419e2c6d86 100644 --- a/arch/arm/lib/copy_from_user.S +++ b/arch/arm/lib/copy_from_user.S @@ -91,7 +91,7 @@ .endm .macro str1b ptr reg cond=al abort - str\cond\()b \reg, [\ptr], #1 + strb\cond \reg, [\ptr], #1 .endm .macro enter reg1 reg2 diff --git a/arch/arm/lib/copy_page.S b/arch/arm/lib/copy_page.S index 6ee2f6706f869..b84ce17920439 100644 --- a/arch/arm/lib/copy_page.S +++ b/arch/arm/lib/copy_page.S @@ -39,9 +39,9 @@ ENTRY(copy_page) .endr subs r2, r2, #1 @ 1 stmia r0!, {r3, r4, ip, lr} @ 4 - ldmgtia r1!, {r3, r4, ip, lr} @ 4 + ldmiagt r1!, {r3, r4, ip, lr} @ 4 bgt 1b @ 1 - PLD( ldmeqia r1!, {r3, r4, ip, lr} ) + PLD( ldmiaeq r1!, {r3, r4, ip, lr} ) PLD( beq 2b ) ldmfd sp!, {r4, pc} @ 3 ENDPROC(copy_page) diff --git a/arch/arm/lib/copy_template.S b/arch/arm/lib/copy_template.S index 652e4d98cd47b..a11f2c25e03a7 100644 --- a/arch/arm/lib/copy_template.S +++ b/arch/arm/lib/copy_template.S @@ -99,7 +99,7 @@ CALGN( ands ip, r0, #31 ) CALGN( rsb r3, ip, #32 ) - CALGN( sbcnes r4, r3, r2 ) @ C is always set here + CALGN( sbcsne r4, r3, r2 ) @ C is always set here CALGN( bcs 2f ) CALGN( adr r4, 6f ) CALGN( subs r2, r2, r3 ) @ C gets set @@ -204,7 +204,7 @@ CALGN( ands ip, r0, #31 ) CALGN( rsb ip, ip, #32 ) - CALGN( sbcnes r4, ip, r2 ) @ C is always set here + CALGN( sbcsne r4, ip, r2 ) @ C is always set here CALGN( subcc r2, r2, ip ) CALGN( bcc 15f ) @@ -241,7 +241,7 @@ orr r9, r9, ip, lspush #\push mov ip, ip, lspull #\pull orr ip, ip, lr, lspush #\push - str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f + str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, abort=19f bge 12b PLD( cmn r2, #96 ) PLD( bge 13b ) diff --git a/arch/arm/lib/copy_to_user.S b/arch/arm/lib/copy_to_user.S index 97a6ff4b7e3ca..c7d08096e3545 100644 --- a/arch/arm/lib/copy_to_user.S +++ b/arch/arm/lib/copy_to_user.S @@ -49,7 +49,7 @@ .endm .macro ldr1b ptr reg cond=al abort - ldr\cond\()b \reg, [\ptr], #1 + ldrb\cond \reg, [\ptr], #1 .endm #ifdef CONFIG_CPU_USE_DOMAINS diff --git a/arch/arm/lib/csumpartial.S b/arch/arm/lib/csumpartial.S index 984e0f29d548b..bd84e2db353b1 100644 --- a/arch/arm/lib/csumpartial.S +++ b/arch/arm/lib/csumpartial.S @@ -40,9 +40,9 @@ td3 .req lr /* we must have at least one byte. */ tst buf, #1 @ odd address? movne sum, sum, ror #8 - ldrneb td0, [buf], #1 + ldrbne td0, [buf], #1 subne len, len, #1 - adcnes sum, sum, td0, put_byte_1 + adcsne sum, sum, td0, put_byte_1 .Lless4: tst len, #6 beq .Lless8_byte @@ -68,8 +68,8 @@ td3 .req lr bne .Lless8_wordlp .Lless8_byte: tst len, #1 @ odd number of bytes - ldrneb td0, [buf], #1 @ include last byte - adcnes sum, sum, td0, put_byte_0 @ update checksum + ldrbne td0, [buf], #1 @ include last byte + adcsne sum, sum, td0, put_byte_0 @ update checksum .Ldone: adc r0, sum, #0 @ collect up the last carry ldr td0, [sp], #4 @@ -78,17 +78,17 @@ td3 .req lr ldr pc, [sp], #4 @ return .Lnot_aligned: tst buf, #1 @ odd address - ldrneb td0, [buf], #1 @ make even + ldrbne td0, [buf], #1 @ make even subne len, len, #1 - adcnes sum, sum, td0, put_byte_1 @ update checksum + adcsne sum, sum, td0, put_byte_1 @ update checksum tst buf, #2 @ 32-bit aligned? #if __LINUX_ARM_ARCH__ >= 4 - ldrneh td0, [buf], #2 @ make 32-bit aligned + ldrhne td0, [buf], #2 @ make 32-bit aligned subne len, len, #2 #else - ldrneb td0, [buf], #1 - ldrneb ip, [buf], #1 + ldrbne td0, [buf], #1 + ldrbne ip, [buf], #1 subne len, len, #2 #ifndef __ARMEB__ orrne td0, td0, ip, lsl #8 @@ -96,7 +96,7 @@ td3 .req lr orrne td0, ip, td0, lsl #8 #endif #endif - adcnes sum, sum, td0 @ update checksum + adcsne sum, sum, td0 @ update checksum ret lr ENTRY(csum_partial) diff --git a/arch/arm/lib/csumpartialcopygeneric.S b/arch/arm/lib/csumpartialcopygeneric.S index 10b45909610ca..08e17758cbea9 100644 --- a/arch/arm/lib/csumpartialcopygeneric.S +++ b/arch/arm/lib/csumpartialcopygeneric.S @@ -148,9 +148,9 @@ FN_ENTRY strb r5, [dst], #1 mov r5, r4, get_byte_2 .Lexit: tst len, #1 - strneb r5, [dst], #1 + strbne r5, [dst], #1 andne r5, r5, #255 - adcnes sum, sum, r5, put_byte_0 + adcsne sum, sum, r5, put_byte_0 /* * If the dst pointer was not 16-bit aligned, we diff --git a/arch/arm/lib/csumpartialcopyuser.S b/arch/arm/lib/csumpartialcopyuser.S index b83fdc06286a6..f4716d98e0b4a 100644 --- a/arch/arm/lib/csumpartialcopyuser.S +++ b/arch/arm/lib/csumpartialcopyuser.S @@ -95,7 +95,7 @@ add r2, r2, r1 mov r0, #0 @ zero the buffer 9002: teq r2, r1 - strneb r0, [r1], #1 + strbne r0, [r1], #1 bne 9002b load_regs .popsection diff --git a/arch/arm/lib/div64.S b/arch/arm/lib/div64.S index a9eafe4981eb8..4d80f690c48bf 100644 --- a/arch/arm/lib/div64.S +++ b/arch/arm/lib/div64.S @@ -88,8 +88,8 @@ UNWIND(.fnstart) @ Break out early if dividend reaches 0. 2: cmp xh, yl orrcs yh, yh, ip - subcss xh, xh, yl - movnes ip, ip, lsr #1 + subscs xh, xh, yl + movsne ip, ip, lsr #1 mov yl, yl, lsr #1 bne 2b diff --git a/arch/arm/lib/floppydma.S b/arch/arm/lib/floppydma.S index 617150b1baef0..de68d3b343e30 100644 --- a/arch/arm/lib/floppydma.S +++ b/arch/arm/lib/floppydma.S @@ -14,8 +14,8 @@ .global floppy_fiqin_end ENTRY(floppy_fiqin_start) subs r9, r9, #1 - ldrgtb r12, [r11, #-4] - ldrleb r12, [r11], #0 + ldrbgt r12, [r11, #-4] + ldrble r12, [r11], #0 strb r12, [r10], #1 subs pc, lr, #4 floppy_fiqin_end: @@ -23,10 +23,10 @@ floppy_fiqin_end: .global floppy_fiqout_end ENTRY(floppy_fiqout_start) subs r9, r9, #1 - ldrgeb r12, [r10], #1 + ldrbge r12, [r10], #1 movlt r12, #0 - strleb r12, [r11], #0 - subles pc, lr, #4 + strble r12, [r11], #0 + subsle pc, lr, #4 strb r12, [r11, #-4] subs pc, lr, #4 floppy_fiqout_end: diff --git a/arch/arm/lib/io-readsb.S b/arch/arm/lib/io-readsb.S index c31b2f3153f17..91038a0a77b57 100644 --- a/arch/arm/lib/io-readsb.S +++ b/arch/arm/lib/io-readsb.S @@ -16,10 +16,10 @@ cmp ip, #2 ldrb r3, [r0] strb r3, [r1], #1 - ldrgeb r3, [r0] - strgeb r3, [r1], #1 - ldrgtb r3, [r0] - strgtb r3, [r1], #1 + ldrbge r3, [r0] + strbge r3, [r1], #1 + ldrbgt r3, [r0] + strbgt r3, [r1], #1 subs r2, r2, ip bne .Linsb_aligned @@ -72,7 +72,7 @@ ENTRY(__raw_readsb) bpl .Linsb_16_lp tst r2, #15 - ldmeqfd sp!, {r4 - r6, pc} + ldmfdeq sp!, {r4 - r6, pc} .Linsb_no_16: tst r2, #8 beq .Linsb_no_8 @@ -109,15 +109,15 @@ ENTRY(__raw_readsb) str r3, [r1], #4 .Linsb_no_4: ands r2, r2, #3 - ldmeqfd sp!, {r4 - r6, pc} + ldmfdeq sp!, {r4 - r6, pc} cmp r2, #2 ldrb r3, [r0] strb r3, [r1], #1 - ldrgeb r3, [r0] - strgeb r3, [r1], #1 - ldrgtb r3, [r0] - strgtb r3, [r1] + ldrbge r3, [r0] + strbge r3, [r1], #1 + ldrbgt r3, [r0] + strbgt r3, [r1] ldmfd sp!, {r4 - r6, pc} ENDPROC(__raw_readsb) diff --git a/arch/arm/lib/io-readsl.S b/arch/arm/lib/io-readsl.S index 2ed86fa5465f7..f2e2064318d21 100644 --- a/arch/arm/lib/io-readsl.S +++ b/arch/arm/lib/io-readsl.S @@ -30,7 +30,7 @@ ENTRY(__raw_readsl) 2: movs r2, r2, lsl #31 ldrcs r3, [r0, #0] ldrcs ip, [r0, #0] - stmcsia r1!, {r3, ip} + stmiacs r1!, {r3, ip} ldrne r3, [r0, #0] strne r3, [r1, #0] ret lr diff --git a/arch/arm/lib/io-readsw-armv3.S b/arch/arm/lib/io-readsw-armv3.S index 413da99145292..8b25b69c516e7 100644 --- a/arch/arm/lib/io-readsw-armv3.S +++ b/arch/arm/lib/io-readsw-armv3.S @@ -68,7 +68,7 @@ ENTRY(__raw_readsw) bpl .Linsw_8_lp tst r2, #7 - ldmeqfd sp!, {r4, r5, r6, pc} + ldmfdeq sp!, {r4, r5, r6, pc} .Lno_insw_8: tst r2, #4 beq .Lno_insw_4 @@ -97,9 +97,9 @@ ENTRY(__raw_readsw) .Lno_insw_2: tst r2, #1 ldrne r3, [r0] - strneb r3, [r1], #1 + strbne r3, [r1], #1 movne r3, r3, lsr #8 - strneb r3, [r1] + strbne r3, [r1] ldmfd sp!, {r4, r5, r6, pc} diff --git a/arch/arm/lib/io-readsw-armv4.S b/arch/arm/lib/io-readsw-armv4.S index d9a45e9692aee..5efdd66f5dcd6 100644 --- a/arch/arm/lib/io-readsw-armv4.S +++ b/arch/arm/lib/io-readsw-armv4.S @@ -76,8 +76,8 @@ ENTRY(__raw_readsw) pack r3, r3, ip str r3, [r1], #4 -.Lno_insw_2: ldrneh r3, [r0] - strneh r3, [r1] +.Lno_insw_2: ldrhne r3, [r0] + strhne r3, [r1] ldmfd sp!, {r4, r5, pc} @@ -94,7 +94,7 @@ ENTRY(__raw_readsw) #endif .Linsw_noalign: stmfd sp!, {r4, lr} - ldrccb ip, [r1, #-1]! + ldrbcc ip, [r1, #-1]! bcc 1f ldrh ip, [r0] @@ -121,11 +121,11 @@ ENTRY(__raw_readsw) 3: tst r2, #1 strb ip, [r1], #1 - ldrneh ip, [r0] + ldrhne ip, [r0] _BE_ONLY_( movne ip, ip, ror #8 ) - strneb ip, [r1], #1 + strbne ip, [r1], #1 _LE_ONLY_( movne ip, ip, lsr #8 ) _BE_ONLY_( movne ip, ip, lsr #24 ) - strneb ip, [r1] + strbne ip, [r1] ldmfd sp!, {r4, pc} ENDPROC(__raw_readsw) diff --git a/arch/arm/lib/io-writesb.S b/arch/arm/lib/io-writesb.S index a46bbc9b168b4..7d2881a2381eb 100644 --- a/arch/arm/lib/io-writesb.S +++ b/arch/arm/lib/io-writesb.S @@ -36,10 +36,10 @@ cmp ip, #2 ldrb r3, [r1], #1 strb r3, [r0] - ldrgeb r3, [r1], #1 - strgeb r3, [r0] - ldrgtb r3, [r1], #1 - strgtb r3, [r0] + ldrbge r3, [r1], #1 + strbge r3, [r0] + ldrbgt r3, [r1], #1 + strbgt r3, [r0] subs r2, r2, ip bne .Loutsb_aligned @@ -64,7 +64,7 @@ ENTRY(__raw_writesb) bpl .Loutsb_16_lp tst r2, #15 - ldmeqfd sp!, {r4, r5, pc} + ldmfdeq sp!, {r4, r5, pc} .Loutsb_no_16: tst r2, #8 beq .Loutsb_no_8 @@ -80,15 +80,15 @@ ENTRY(__raw_writesb) outword r3 .Loutsb_no_4: ands r2, r2, #3 - ldmeqfd sp!, {r4, r5, pc} + ldmfdeq sp!, {r4, r5, pc} cmp r2, #2 ldrb r3, [r1], #1 strb r3, [r0] - ldrgeb r3, [r1], #1 - strgeb r3, [r0] - ldrgtb r3, [r1] - strgtb r3, [r0] + ldrbge r3, [r1], #1 + strbge r3, [r0] + ldrbgt r3, [r1] + strbgt r3, [r0] ldmfd sp!, {r4, r5, pc} ENDPROC(__raw_writesb) diff --git a/arch/arm/lib/io-writesl.S b/arch/arm/lib/io-writesl.S index 4ea2435988c1f..7596ac0c90b05 100644 --- a/arch/arm/lib/io-writesl.S +++ b/arch/arm/lib/io-writesl.S @@ -28,7 +28,7 @@ ENTRY(__raw_writesl) bpl 1b ldmfd sp!, {r4, lr} 2: movs r2, r2, lsl #31 - ldmcsia r1!, {r3, ip} + ldmiacs r1!, {r3, ip} strcs r3, [r0, #0] ldrne r3, [r1, #0] strcs ip, [r0, #0] diff --git a/arch/arm/lib/io-writesw-armv3.S b/arch/arm/lib/io-writesw-armv3.S index 121789eb68023..cb94b9b494056 100644 --- a/arch/arm/lib/io-writesw-armv3.S +++ b/arch/arm/lib/io-writesw-armv3.S @@ -79,7 +79,7 @@ ENTRY(__raw_writesw) bpl .Loutsw_8_lp tst r2, #7 - ldmeqfd sp!, {r4, r5, r6, pc} + ldmfdeq sp!, {r4, r5, r6, pc} .Lno_outsw_8: tst r2, #4 beq .Lno_outsw_4 diff --git a/arch/arm/lib/io-writesw-armv4.S b/arch/arm/lib/io-writesw-armv4.S index 269f90c51ad27..e6645b2f249ef 100644 --- a/arch/arm/lib/io-writesw-armv4.S +++ b/arch/arm/lib/io-writesw-armv4.S @@ -61,8 +61,8 @@ ENTRY(__raw_writesw) ldr r3, [r1], #4 outword r3 -.Lno_outsw_2: ldrneh r3, [r1] - strneh r3, [r0] +.Lno_outsw_2: ldrhne r3, [r1] + strhne r3, [r0] ldmfd sp!, {r4, r5, pc} @@ -95,6 +95,6 @@ ENTRY(__raw_writesw) tst r2, #1 3: movne ip, r3, lsr #8 - strneh ip, [r0] + strhne ip, [r0] ret lr ENDPROC(__raw_writesw) diff --git a/arch/arm/lib/lib1funcs.S b/arch/arm/lib/lib1funcs.S index 9397b2e532afa..c23f9d9e29704 100644 --- a/arch/arm/lib/lib1funcs.S +++ b/arch/arm/lib/lib1funcs.S @@ -96,7 +96,7 @@ Boston, MA 02111-1307, USA. */ subhs \dividend, \dividend, \divisor, lsr #3 orrhs \result, \result, \curbit, lsr #3 cmp \dividend, #0 @ Early termination? - movnes \curbit, \curbit, lsr #4 @ No, any more bits to do? + movsne \curbit, \curbit, lsr #4 @ No, any more bits to do? movne \divisor, \divisor, lsr #4 bne 1b @@ -182,7 +182,7 @@ Boston, MA 02111-1307, USA. */ subhs \dividend, \dividend, \divisor, lsr #3 cmp \dividend, #1 mov \divisor, \divisor, lsr #4 - subges \order, \order, #4 + subsge \order, \order, #4 bge 1b tst \order, #3 diff --git a/arch/arm/lib/memcpy.S b/arch/arm/lib/memcpy.S index 64111bd4440b1..4a6997bb44043 100644 --- a/arch/arm/lib/memcpy.S +++ b/arch/arm/lib/memcpy.S @@ -30,7 +30,7 @@ .endm .macro ldr1b ptr reg cond=al abort - ldr\cond\()b \reg, [\ptr], #1 + ldrb\cond \reg, [\ptr], #1 .endm .macro str1w ptr reg abort @@ -42,7 +42,7 @@ .endm .macro str1b ptr reg cond=al abort - str\cond\()b \reg, [\ptr], #1 + strb\cond \reg, [\ptr], #1 .endm .macro enter reg1 reg2 diff --git a/arch/arm/lib/memmove.S b/arch/arm/lib/memmove.S index 69a9d47fc5abd..d70304cb2cd0d 100644 --- a/arch/arm/lib/memmove.S +++ b/arch/arm/lib/memmove.S @@ -59,7 +59,7 @@ ENTRY(memmove) blt 5f CALGN( ands ip, r0, #31 ) - CALGN( sbcnes r4, ip, r2 ) @ C is always set here + CALGN( sbcsne r4, ip, r2 ) @ C is always set here CALGN( bcs 2f ) CALGN( adr r4, 6f ) CALGN( subs r2, r2, ip ) @ C is set here @@ -114,20 +114,20 @@ ENTRY(memmove) UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block 8: movs r2, r2, lsl #31 - ldrneb r3, [r1, #-1]! - ldrcsb r4, [r1, #-1]! - ldrcsb ip, [r1, #-1] - strneb r3, [r0, #-1]! - strcsb r4, [r0, #-1]! - strcsb ip, [r0, #-1] + ldrbne r3, [r1, #-1]! + ldrbcs r4, [r1, #-1]! + ldrbcs ip, [r1, #-1] + strbne r3, [r0, #-1]! + strbcs r4, [r0, #-1]! + strbcs ip, [r0, #-1] ldmfd sp!, {r0, r4, pc} 9: cmp ip, #2 - ldrgtb r3, [r1, #-1]! - ldrgeb r4, [r1, #-1]! + ldrbgt r3, [r1, #-1]! + ldrbge r4, [r1, #-1]! ldrb lr, [r1, #-1]! - strgtb r3, [r0, #-1]! - strgeb r4, [r0, #-1]! + strbgt r3, [r0, #-1]! + strbge r4, [r0, #-1]! subs r2, r2, ip strb lr, [r0, #-1]! blt 8b @@ -150,7 +150,7 @@ ENTRY(memmove) blt 14f CALGN( ands ip, r0, #31 ) - CALGN( sbcnes r4, ip, r2 ) @ C is always set here + CALGN( sbcsne r4, ip, r2 ) @ C is always set here CALGN( subcc r2, r2, ip ) CALGN( bcc 15f ) diff --git a/arch/arm/lib/memset.S b/arch/arm/lib/memset.S index ed6d35d9cdb5a..5593a45e0a8c6 100644 --- a/arch/arm/lib/memset.S +++ b/arch/arm/lib/memset.S @@ -44,20 +44,20 @@ UNWIND( .save {r8, lr} ) mov lr, r3 2: subs r2, r2, #64 - stmgeia ip!, {r1, r3, r8, lr} @ 64 bytes at a time. - stmgeia ip!, {r1, r3, r8, lr} - stmgeia ip!, {r1, r3, r8, lr} - stmgeia ip!, {r1, r3, r8, lr} + stmiage ip!, {r1, r3, r8, lr} @ 64 bytes at a time. + stmiage ip!, {r1, r3, r8, lr} + stmiage ip!, {r1, r3, r8, lr} + stmiage ip!, {r1, r3, r8, lr} bgt 2b - ldmeqfd sp!, {r8, pc} @ Now <64 bytes to go. + ldmfdeq sp!, {r8, pc} @ Now <64 bytes to go. /* * No need to correct the count; we're only testing bits from now on */ tst r2, #32 - stmneia ip!, {r1, r3, r8, lr} - stmneia ip!, {r1, r3, r8, lr} + stmiane ip!, {r1, r3, r8, lr} + stmiane ip!, {r1, r3, r8, lr} tst r2, #16 - stmneia ip!, {r1, r3, r8, lr} + stmiane ip!, {r1, r3, r8, lr} ldmfd sp!, {r8, lr} UNWIND( .fnend ) @@ -87,22 +87,22 @@ UNWIND( .save {r4-r8, lr} ) rsb r8, r8, #32 sub r2, r2, r8 movs r8, r8, lsl #(32 - 4) - stmcsia ip!, {r4, r5, r6, r7} - stmmiia ip!, {r4, r5} + stmiacs ip!, {r4, r5, r6, r7} + stmiami ip!, {r4, r5} tst r8, #(1 << 30) mov r8, r1 strne r1, [ip], #4 3: subs r2, r2, #64 - stmgeia ip!, {r1, r3-r8, lr} - stmgeia ip!, {r1, r3-r8, lr} + stmiage ip!, {r1, r3-r8, lr} + stmiage ip!, {r1, r3-r8, lr} bgt 3b - ldmeqfd sp!, {r4-r8, pc} + ldmfdeq sp!, {r4-r8, pc} tst r2, #32 - stmneia ip!, {r1, r3-r8, lr} + stmiane ip!, {r1, r3-r8, lr} tst r2, #16 - stmneia ip!, {r4-r7} + stmiane ip!, {r4-r7} ldmfd sp!, {r4-r8, lr} UNWIND( .fnend ) @@ -110,7 +110,7 @@ UNWIND( .fnend ) UNWIND( .fnstart ) 4: tst r2, #8 - stmneia ip!, {r1, r3} + stmiane ip!, {r1, r3} tst r2, #4 strne r1, [ip], #4 /* @@ -118,17 +118,17 @@ UNWIND( .fnstart ) * may have an unaligned pointer as well. */ 5: tst r2, #2 - strneb r1, [ip], #1 - strneb r1, [ip], #1 + strbne r1, [ip], #1 + strbne r1, [ip], #1 tst r2, #1 - strneb r1, [ip], #1 + strbne r1, [ip], #1 ret lr 6: subs r2, r2, #4 @ 1 do we have enough blt 5b @ 1 bytes to align with? cmp r3, #2 @ 1 - strltb r1, [ip], #1 @ 1 - strleb r1, [ip], #1 @ 1 + strblt r1, [ip], #1 @ 1 + strble r1, [ip], #1 @ 1 strb r1, [ip], #1 @ 1 add r2, r2, r3 @ 1 (r2 = r2 - (4 - r3)) b 1b diff --git a/arch/arm/lib/xor-neon.c b/arch/arm/lib/xor-neon.c index 2c40aeab3eaae..c691b901092f5 100644 --- a/arch/arm/lib/xor-neon.c +++ b/arch/arm/lib/xor-neon.c @@ -14,7 +14,7 @@ MODULE_LICENSE("GPL"); #ifndef __ARM_NEON__ -#error You should compile this file with '-mfloat-abi=softfp -mfpu=neon' +#error You should compile this file with '-march=armv7-a -mfloat-abi=softfp -mfpu=neon' #endif /* diff --git a/arch/arm/mach-actions/platsmp.c b/arch/arm/mach-actions/platsmp.c index 3efaa10efc439..4fd479c948e67 100644 --- a/arch/arm/mach-actions/platsmp.c +++ b/arch/arm/mach-actions/platsmp.c @@ -39,10 +39,6 @@ static void __iomem *sps_base_addr; static void __iomem *timer_base_addr; static int ncores; -static DEFINE_SPINLOCK(boot_lock); - -void owl_secondary_startup(void); - static int s500_wakeup_secondary(unsigned int cpu) { int ret; @@ -84,7 +80,6 @@ static int s500_wakeup_secondary(unsigned int cpu) static int s500_smp_boot_secondary(unsigned int cpu, struct task_struct *idle) { - unsigned long timeout; int ret; ret = s500_wakeup_secondary(cpu); @@ -93,21 +88,11 @@ static int s500_smp_boot_secondary(unsigned int cpu, struct task_struct *idle) udelay(10); - spin_lock(&boot_lock); - smp_send_reschedule(cpu); - timeout = jiffies + (1 * HZ); - while (time_before(jiffies, timeout)) { - if (pen_release == -1) - break; - } - writel(0, timer_base_addr + OWL_CPU1_ADDR + (cpu - 1) * 4); writel(0, timer_base_addr + OWL_CPU1_FLAG + (cpu - 1) * 4); - spin_unlock(&boot_lock); - return 0; } diff --git a/arch/arm/mach-exynos/headsmp.S b/arch/arm/mach-exynos/headsmp.S index 005695c9bf400..0ac2cb9a73556 100644 --- a/arch/arm/mach-exynos/headsmp.S +++ b/arch/arm/mach-exynos/headsmp.S @@ -36,4 +36,4 @@ ENDPROC(exynos4_secondary_startup) .align 2 1: .long . - .long pen_release + .long exynos_pen_release diff --git a/arch/arm/mach-exynos/platsmp.c b/arch/arm/mach-exynos/platsmp.c index b6da7edbbd2fd..abcac61642331 100644 --- a/arch/arm/mach-exynos/platsmp.c +++ b/arch/arm/mach-exynos/platsmp.c @@ -28,6 +28,9 @@ extern void exynos4_secondary_startup(void); +/* XXX exynos_pen_release is cargo culted code - DO NOT COPY XXX */ +volatile int exynos_pen_release = -1; + #ifdef CONFIG_HOTPLUG_CPU static inline void cpu_leave_lowpower(u32 core_id) { @@ -57,7 +60,7 @@ static inline void platform_do_lowpower(unsigned int cpu, int *spurious) wfi(); - if (pen_release == core_id) { + if (exynos_pen_release == core_id) { /* * OK, proper wakeup, we're done */ @@ -228,15 +231,17 @@ void exynos_core_restart(u32 core_id) } /* - * Write pen_release in a way that is guaranteed to be visible to all - * observers, irrespective of whether they're taking part in coherency + * XXX CARGO CULTED CODE - DO NOT COPY XXX + * + * Write exynos_pen_release in a way that is guaranteed to be visible to + * all observers, irrespective of whether they're taking part in coherency * or not. This is necessary for the hotplug code to work reliably. */ -static void write_pen_release(int val) +static void exynos_write_pen_release(int val) { - pen_release = val; + exynos_pen_release = val; smp_wmb(); - sync_cache_w(&pen_release); + sync_cache_w(&exynos_pen_release); } static DEFINE_SPINLOCK(boot_lock); @@ -247,7 +252,7 @@ static void exynos_secondary_init(unsigned int cpu) * let the primary processor know we're out of the * pen, then head off into the C entry point */ - write_pen_release(-1); + exynos_write_pen_release(-1); /* * Synchronise with the boot thread. @@ -322,12 +327,12 @@ static int exynos_boot_secondary(unsigned int cpu, struct task_struct *idle) /* * The secondary processor is waiting to be released from * the holding pen - release it, then wait for it to flag - * that it has been released by resetting pen_release. + * that it has been released by resetting exynos_pen_release. * - * Note that "pen_release" is the hardware CPU core ID, whereas + * Note that "exynos_pen_release" is the hardware CPU core ID, whereas * "cpu" is Linux's internal ID. */ - write_pen_release(core_id); + exynos_write_pen_release(core_id); if (!exynos_cpu_power_state(core_id)) { exynos_cpu_power_up(core_id); @@ -376,13 +381,13 @@ static int exynos_boot_secondary(unsigned int cpu, struct task_struct *idle) else arch_send_wakeup_ipi_mask(cpumask_of(cpu)); - if (pen_release == -1) + if (exynos_pen_release == -1) break; udelay(10); } - if (pen_release != -1) + if (exynos_pen_release != -1) ret = -ETIMEDOUT; /* @@ -392,7 +397,7 @@ static int exynos_boot_secondary(unsigned int cpu, struct task_struct *idle) fail: spin_unlock(&boot_lock); - return pen_release != -1 ? ret : 0; + return exynos_pen_release != -1 ? ret : 0; } static void __init exynos_smp_prepare_cpus(unsigned int max_cpus) diff --git a/arch/arm/mach-ks8695/include/mach/entry-macro.S b/arch/arm/mach-ks8695/include/mach/entry-macro.S index 8315b34f32ff0..7ff812cb010bb 100644 --- a/arch/arm/mach-ks8695/include/mach/entry-macro.S +++ b/arch/arm/mach-ks8695/include/mach/entry-macro.S @@ -42,6 +42,6 @@ moveq \irqstat, \irqstat, lsr #2 addeq \irqnr, \irqnr, #2 tst \irqstat, #0x01 - addeqs \irqnr, \irqnr, #1 + addseq \irqnr, \irqnr, #1 1001: .endm diff --git a/arch/arm/mach-omap2/omap_hwmod.c b/arch/arm/mach-omap2/omap_hwmod.c index b5531dd3ae9c3..3a04c73ac03c3 100644 --- a/arch/arm/mach-omap2/omap_hwmod.c +++ b/arch/arm/mach-omap2/omap_hwmod.c @@ -1002,8 +1002,10 @@ static int _enable_clocks(struct omap_hwmod *oh) clk_enable(oh->_clk); list_for_each_entry(os, &oh->slave_ports, node) { - if (os->_clk && (os->flags & OCPIF_SWSUP_IDLE)) + if (os->_clk && (os->flags & OCPIF_SWSUP_IDLE)) { + omap2_clk_deny_idle(os->_clk); clk_enable(os->_clk); + } } /* The opt clocks are controlled by the device driver. */ @@ -1055,8 +1057,10 @@ static int _disable_clocks(struct omap_hwmod *oh) clk_disable(oh->_clk); list_for_each_entry(os, &oh->slave_ports, node) { - if (os->_clk && (os->flags & OCPIF_SWSUP_IDLE)) + if (os->_clk && (os->flags & OCPIF_SWSUP_IDLE)) { clk_disable(os->_clk); + omap2_clk_allow_idle(os->_clk); + } } if (oh->flags & HWMOD_OPT_CLKS_NEEDED) @@ -2436,9 +2440,13 @@ static void _setup_iclk_autoidle(struct omap_hwmod *oh) continue; if (os->flags & OCPIF_SWSUP_IDLE) { - /* XXX omap_iclk_deny_idle(c); */ + /* + * we might have multiple users of one iclk with + * different requirements, disable autoidle when + * the module is enabled, e.g. dss iclk + */ } else { - /* XXX omap_iclk_allow_idle(c); */ + /* we are enabling autoidle afterwards anyways */ clk_enable(os->_clk); } } diff --git a/arch/arm/mach-omap2/prm_common.c b/arch/arm/mach-omap2/prm_common.c index 058a37e6d11c3..fd6e0671f9573 100644 --- a/arch/arm/mach-omap2/prm_common.c +++ b/arch/arm/mach-omap2/prm_common.c @@ -523,8 +523,10 @@ void omap_prm_reset_system(void) prm_ll_data->reset_system(); - while (1) + while (1) { cpu_relax(); + wfe(); + } } /** diff --git a/arch/arm/mach-oxnas/Makefile b/arch/arm/mach-oxnas/Makefile index b625906a99702..61a34e1c0f221 100644 --- a/arch/arm/mach-oxnas/Makefile +++ b/arch/arm/mach-oxnas/Makefile @@ -1,2 +1 @@ obj-$(CONFIG_SMP) += platsmp.o headsmp.o -obj-$(CONFIG_HOTPLUG_CPU) += hotplug.o diff --git a/arch/arm/mach-oxnas/hotplug.c b/arch/arm/mach-oxnas/hotplug.c deleted file mode 100644 index 854f29b8cba6e..0000000000000 --- a/arch/arm/mach-oxnas/hotplug.c +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2002 ARM Ltd. - * All Rights Reserved - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ -#include <linux/kernel.h> -#include <linux/errno.h> -#include <linux/smp.h> - -#include <asm/cp15.h> -#include <asm/smp_plat.h> - -static inline void cpu_enter_lowpower(void) -{ - unsigned int v; - - asm volatile( - " mcr p15, 0, %1, c7, c5, 0\n" - " mcr p15, 0, %1, c7, c10, 4\n" - /* - * Turn off coherency - */ - " mrc p15, 0, %0, c1, c0, 1\n" - " bic %0, %0, #0x20\n" - " mcr p15, 0, %0, c1, c0, 1\n" - " mrc p15, 0, %0, c1, c0, 0\n" - " bic %0, %0, %2\n" - " mcr p15, 0, %0, c1, c0, 0\n" - : "=&r" (v) - : "r" (0), "Ir" (CR_C) - : "cc"); -} - -static inline void cpu_leave_lowpower(void) -{ - unsigned int v; - - asm volatile( "mrc p15, 0, %0, c1, c0, 0\n" - " orr %0, %0, %1\n" - " mcr p15, 0, %0, c1, c0, 0\n" - " mrc p15, 0, %0, c1, c0, 1\n" - " orr %0, %0, #0x20\n" - " mcr p15, 0, %0, c1, c0, 1\n" - : "=&r" (v) - : "Ir" (CR_C) - : "cc"); -} - -static inline void platform_do_lowpower(unsigned int cpu, int *spurious) -{ - /* - * there is no power-control hardware on this platform, so all - * we can do is put the core into WFI; this is safe as the calling - * code will have already disabled interrupts - */ - for (;;) { - /* - * here's the WFI - */ - asm(".word 0xe320f003\n" - : - : - : "memory", "cc"); - - if (pen_release == cpu_logical_map(cpu)) { - /* - * OK, proper wakeup, we're done - */ - break; - } - - /* - * Getting here, means that we have come out of WFI without - * having been woken up - this shouldn't happen - * - * Just note it happening - when we're woken, we can report - * its occurrence. - */ - (*spurious)++; - } -} - -/* - * platform-specific code to shutdown a CPU - * - * Called with IRQs disabled - */ -void ox820_cpu_die(unsigned int cpu) -{ - int spurious = 0; - - /* - * we're ready for shutdown now, so do it - */ - cpu_enter_lowpower(); - platform_do_lowpower(cpu, &spurious); - - /* - * bring this CPU back into the world of cache - * coherency, and then restore interrupts - */ - cpu_leave_lowpower(); - - if (spurious) - pr_warn("CPU%u: %u spurious wakeup calls\n", cpu, spurious); -} diff --git a/arch/arm/mach-oxnas/platsmp.c b/arch/arm/mach-oxnas/platsmp.c index 442cc8a2f7dc8..735141c0e3a37 100644 --- a/arch/arm/mach-oxnas/platsmp.c +++ b/arch/arm/mach-oxnas/platsmp.c @@ -19,7 +19,6 @@ #include <asm/smp_scu.h> extern void ox820_secondary_startup(void); -extern void ox820_cpu_die(unsigned int cpu); static void __iomem *cpu_ctrl; static void __iomem *gic_cpu_ctrl; @@ -94,9 +93,6 @@ unmap_scu: static const struct smp_operations ox820_smp_ops __initconst = { .smp_prepare_cpus = ox820_smp_prepare_cpus, .smp_boot_secondary = ox820_boot_secondary, -#ifdef CONFIG_HOTPLUG_CPU - .cpu_die = ox820_cpu_die, -#endif }; CPU_METHOD_OF_DECLARE(ox820_smp, "oxsemi,ox820-smp", &ox820_smp_ops); diff --git a/arch/arm/mach-prima2/common.h b/arch/arm/mach-prima2/common.h index 6d77b622d1685..457eb7b181600 100644 --- a/arch/arm/mach-prima2/common.h +++ b/arch/arm/mach-prima2/common.h @@ -15,6 +15,8 @@ #include <asm/mach/time.h> #include <asm/exception.h> +extern volatile int prima2_pen_release; + extern const struct smp_operations sirfsoc_smp_ops; extern void sirfsoc_secondary_startup(void); extern void sirfsoc_cpu_die(unsigned int cpu); diff --git a/arch/arm/mach-prima2/headsmp.S b/arch/arm/mach-prima2/headsmp.S index 209d9fc5c16cf..6cf4fc60347b5 100644 --- a/arch/arm/mach-prima2/headsmp.S +++ b/arch/arm/mach-prima2/headsmp.S @@ -34,4 +34,4 @@ ENDPROC(sirfsoc_secondary_startup) .align 1: .long . - .long pen_release + .long prima2_pen_release diff --git a/arch/arm/mach-prima2/hotplug.c b/arch/arm/mach-prima2/hotplug.c index a728c78b996f7..b6cf1527e3309 100644 --- a/arch/arm/mach-prima2/hotplug.c +++ b/arch/arm/mach-prima2/hotplug.c @@ -11,6 +11,7 @@ #include <linux/smp.h> #include <asm/smp_plat.h> +#include "common.h" static inline void platform_do_lowpower(unsigned int cpu) { @@ -18,7 +19,7 @@ static inline void platform_do_lowpower(unsigned int cpu) for (;;) { __asm__ __volatile__("dsb\n\t" "wfi\n\t" : : : "memory"); - if (pen_release == cpu_logical_map(cpu)) { + if (prima2_pen_release == cpu_logical_map(cpu)) { /* * OK, proper wakeup, we're done */ diff --git a/arch/arm/mach-prima2/platsmp.c b/arch/arm/mach-prima2/platsmp.c index 75ef5d4be554c..d1f8b5168083c 100644 --- a/arch/arm/mach-prima2/platsmp.c +++ b/arch/arm/mach-prima2/platsmp.c @@ -24,13 +24,16 @@ static void __iomem *clk_base; static DEFINE_SPINLOCK(boot_lock); +/* XXX prima2_pen_release is cargo culted code - DO NOT COPY XXX */ +volatile int prima2_pen_release = -1; + static void sirfsoc_secondary_init(unsigned int cpu) { /* * let the primary processor know we're out of the * pen, then head off into the C entry point */ - pen_release = -1; + prima2_pen_release = -1; smp_wmb(); /* @@ -80,13 +83,13 @@ static int sirfsoc_boot_secondary(unsigned int cpu, struct task_struct *idle) /* * The secondary processor is waiting to be released from * the holding pen - release it, then wait for it to flag - * that it has been released by resetting pen_release. + * that it has been released by resetting prima2_pen_release. * - * Note that "pen_release" is the hardware CPU ID, whereas + * Note that "prima2_pen_release" is the hardware CPU ID, whereas * "cpu" is Linux's internal ID. */ - pen_release = cpu_logical_map(cpu); - sync_cache_w(&pen_release); + prima2_pen_release = cpu_logical_map(cpu); + sync_cache_w(&prima2_pen_release); /* * Send the secondary CPU SEV, thereby causing the boot monitor to read @@ -97,7 +100,7 @@ static int sirfsoc_boot_secondary(unsigned int cpu, struct task_struct *idle) timeout = jiffies + (1 * HZ); while (time_before(jiffies, timeout)) { smp_rmb(); - if (pen_release == -1) + if (prima2_pen_release == -1) break; udelay(10); @@ -109,7 +112,7 @@ static int sirfsoc_boot_secondary(unsigned int cpu, struct task_struct *idle) */ spin_unlock(&boot_lock); - return pen_release != -1 ? -ENOSYS : 0; + return prima2_pen_release != -1 ? -ENOSYS : 0; } const struct smp_operations sirfsoc_smp_ops __initconst = { diff --git a/arch/arm/mach-qcom/platsmp.c b/arch/arm/mach-qcom/platsmp.c index 5494c9e0c909b..99a6a5e809e0e 100644 --- a/arch/arm/mach-qcom/platsmp.c +++ b/arch/arm/mach-qcom/platsmp.c @@ -46,8 +46,6 @@ extern void secondary_startup_arm(void); -static DEFINE_SPINLOCK(boot_lock); - #ifdef CONFIG_HOTPLUG_CPU static void qcom_cpu_die(unsigned int cpu) { @@ -55,15 +53,6 @@ static void qcom_cpu_die(unsigned int cpu) } #endif -static void qcom_secondary_init(unsigned int cpu) -{ - /* - * Synchronise with the boot thread. - */ - spin_lock(&boot_lock); - spin_unlock(&boot_lock); -} - static int scss_release_secondary(unsigned int cpu) { struct device_node *node; @@ -281,24 +270,12 @@ static int qcom_boot_secondary(unsigned int cpu, int (*func)(unsigned int)) } /* - * set synchronisation state between this boot processor - * and the secondary one - */ - spin_lock(&boot_lock); - - /* * Send the secondary CPU a soft interrupt, thereby causing * the boot monitor to read the system wide flags register, * and branch to the address found there. */ arch_send_wakeup_ipi_mask(cpumask_of(cpu)); - /* - * now the secondary core is starting up let it run its - * calibrations, then wait for it to finish - */ - spin_unlock(&boot_lock); - return ret; } @@ -334,7 +311,6 @@ static void __init qcom_smp_prepare_cpus(unsigned int max_cpus) static const struct smp_operations smp_msm8660_ops __initconst = { .smp_prepare_cpus = qcom_smp_prepare_cpus, - .smp_secondary_init = qcom_secondary_init, .smp_boot_secondary = msm8660_boot_secondary, #ifdef CONFIG_HOTPLUG_CPU .cpu_die = qcom_cpu_die, @@ -344,7 +320,6 @@ CPU_METHOD_OF_DECLARE(qcom_smp, "qcom,gcc-msm8660", &smp_msm8660_ops); static const struct smp_operations qcom_smp_kpssv1_ops __initconst = { .smp_prepare_cpus = qcom_smp_prepare_cpus, - .smp_secondary_init = qcom_secondary_init, .smp_boot_secondary = kpssv1_boot_secondary, #ifdef CONFIG_HOTPLUG_CPU .cpu_die = qcom_cpu_die, @@ -354,7 +329,6 @@ CPU_METHOD_OF_DECLARE(qcom_smp_kpssv1, "qcom,kpss-acc-v1", &qcom_smp_kpssv1_ops) static const struct smp_operations qcom_smp_kpssv2_ops __initconst = { .smp_prepare_cpus = qcom_smp_prepare_cpus, - .smp_secondary_init = qcom_secondary_init, .smp_boot_secondary = kpssv2_boot_secondary, #ifdef CONFIG_HOTPLUG_CPU .cpu_die = qcom_cpu_die, diff --git a/arch/arm/mach-spear/generic.h b/arch/arm/mach-spear/generic.h index 909b97c0b2371..25b4c5e66e396 100644 --- a/arch/arm/mach-spear/generic.h +++ b/arch/arm/mach-spear/generic.h @@ -20,6 +20,8 @@ #include <asm/mach/time.h> +extern volatile int spear_pen_release; + extern void spear13xx_timer_init(void); extern void spear3xx_timer_init(void); extern struct pl022_ssp_controller pl022_plat_data; diff --git a/arch/arm/mach-spear/headsmp.S b/arch/arm/mach-spear/headsmp.S index c52192dc3d9f9..6e250b6c0aa23 100644 --- a/arch/arm/mach-spear/headsmp.S +++ b/arch/arm/mach-spear/headsmp.S @@ -43,5 +43,5 @@ pen: ldr r7, [r6] .align 1: .long . - .long pen_release + .long spear_pen_release ENDPROC(spear13xx_secondary_startup) diff --git a/arch/arm/mach-spear/hotplug.c b/arch/arm/mach-spear/hotplug.c index 12edd1cf8a12f..0dd84f609627a 100644 --- a/arch/arm/mach-spear/hotplug.c +++ b/arch/arm/mach-spear/hotplug.c @@ -16,6 +16,8 @@ #include <asm/cp15.h> #include <asm/smp_plat.h> +#include "generic.h" + static inline void cpu_enter_lowpower(void) { unsigned int v; @@ -57,7 +59,7 @@ static inline void spear13xx_do_lowpower(unsigned int cpu, int *spurious) for (;;) { wfi(); - if (pen_release == cpu) { + if (spear_pen_release == cpu) { /* * OK, proper wakeup, we're done */ diff --git a/arch/arm/mach-spear/platsmp.c b/arch/arm/mach-spear/platsmp.c index 39038a03836ac..b1ff4bb86f6d8 100644 --- a/arch/arm/mach-spear/platsmp.c +++ b/arch/arm/mach-spear/platsmp.c @@ -20,16 +20,21 @@ #include <mach/spear.h> #include "generic.h" +/* XXX spear_pen_release is cargo culted code - DO NOT COPY XXX */ +volatile int spear_pen_release = -1; + /* - * Write pen_release in a way that is guaranteed to be visible to all - * observers, irrespective of whether they're taking part in coherency + * XXX CARGO CULTED CODE - DO NOT COPY XXX + * + * Write spear_pen_release in a way that is guaranteed to be visible to + * all observers, irrespective of whether they're taking part in coherency * or not. This is necessary for the hotplug code to work reliably. */ -static void write_pen_release(int val) +static void spear_write_pen_release(int val) { - pen_release = val; + spear_pen_release = val; smp_wmb(); - sync_cache_w(&pen_release); + sync_cache_w(&spear_pen_release); } static DEFINE_SPINLOCK(boot_lock); @@ -42,7 +47,7 @@ static void spear13xx_secondary_init(unsigned int cpu) * let the primary processor know we're out of the * pen, then head off into the C entry point */ - write_pen_release(-1); + spear_write_pen_release(-1); /* * Synchronise with the boot thread. @@ -64,17 +69,17 @@ static int spear13xx_boot_secondary(unsigned int cpu, struct task_struct *idle) /* * The secondary processor is waiting to be released from * the holding pen - release it, then wait for it to flag - * that it has been released by resetting pen_release. + * that it has been released by resetting spear_pen_release. * - * Note that "pen_release" is the hardware CPU ID, whereas + * Note that "spear_pen_release" is the hardware CPU ID, whereas * "cpu" is Linux's internal ID. */ - write_pen_release(cpu); + spear_write_pen_release(cpu); timeout = jiffies + (1 * HZ); while (time_before(jiffies, timeout)) { smp_rmb(); - if (pen_release == -1) + if (spear_pen_release == -1) break; udelay(10); @@ -86,7 +91,7 @@ static int spear13xx_boot_secondary(unsigned int cpu, struct task_struct *idle) */ spin_unlock(&boot_lock); - return pen_release != -1 ? -ENOSYS : 0; + return spear_pen_release != -1 ? -ENOSYS : 0; } /* diff --git a/arch/arm/mach-tegra/reset-handler.S b/arch/arm/mach-tegra/reset-handler.S index 805f306fa6f70..e22ccf87eded3 100644 --- a/arch/arm/mach-tegra/reset-handler.S +++ b/arch/arm/mach-tegra/reset-handler.S @@ -172,7 +172,7 @@ after_errata: mov32 r5, TEGRA_IRAM_BASE + TEGRA_IRAM_RESET_HANDLER_OFFSET mov r0, #CPU_NOT_RESETTABLE cmp r10, #0 - strneb r0, [r5, #__tegra20_cpu1_resettable_status_offset] + strbne r0, [r5, #__tegra20_cpu1_resettable_status_offset] 1: #endif diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S index 24659952c2784..be68d62566c7c 100644 --- a/arch/arm/mm/cache-v6.S +++ b/arch/arm/mm/cache-v6.S @@ -215,8 +215,8 @@ v6_dma_inv_range: #endif tst r1, #D_CACHE_LINE_SIZE - 1 #ifdef CONFIG_DMA_CACHE_RWFO - ldrneb r2, [r1, #-1] @ read for ownership - strneb r2, [r1, #-1] @ write for ownership + ldrbne r2, [r1, #-1] @ read for ownership + strbne r2, [r1, #-1] @ write for ownership #endif bic r1, r1, #D_CACHE_LINE_SIZE - 1 #ifdef HARVARD_CACHE @@ -284,8 +284,8 @@ ENTRY(v6_dma_flush_range) add r0, r0, #D_CACHE_LINE_SIZE cmp r0, r1 #ifdef CONFIG_DMA_CACHE_RWFO - ldrlob r2, [r0] @ read for ownership - strlob r2, [r0] @ write for ownership + ldrblo r2, [r0] @ read for ownership + strblo r2, [r0] @ write for ownership #endif blo 1b mov r0, #0 diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c index b03202cddddb2..f74cdce6d4dad 100644 --- a/arch/arm/mm/copypage-v4mc.c +++ b/arch/arm/mm/copypage-v4mc.c @@ -45,6 +45,7 @@ static void mc_copy_user_page(void *from, void *to) int tmp; asm volatile ("\ + .syntax unified\n\ ldmia %0!, {r2, r3, ip, lr} @ 4\n\ 1: mcr p15, 0, %1, c7, c6, 1 @ 1 invalidate D line\n\ stmia %1!, {r2, r3, ip, lr} @ 4\n\ @@ -56,7 +57,7 @@ static void mc_copy_user_page(void *from, void *to) ldmia %0!, {r2, r3, ip, lr} @ 4\n\ subs %2, %2, #1 @ 1\n\ stmia %1!, {r2, r3, ip, lr} @ 4\n\ - ldmneia %0!, {r2, r3, ip, lr} @ 4\n\ + ldmiane %0!, {r2, r3, ip, lr} @ 4\n\ bne 1b @ " : "+&r" (from), "+&r" (to), "=&r" (tmp) : "2" (PAGE_SIZE / 64) diff --git a/arch/arm/mm/copypage-v4wb.c b/arch/arm/mm/copypage-v4wb.c index cd3e165afeede..6d336740aae49 100644 --- a/arch/arm/mm/copypage-v4wb.c +++ b/arch/arm/mm/copypage-v4wb.c @@ -27,6 +27,7 @@ static void v4wb_copy_user_page(void *kto, const void *kfrom) int tmp; asm volatile ("\ + .syntax unified\n\ ldmia %1!, {r3, r4, ip, lr} @ 4\n\ 1: mcr p15, 0, %0, c7, c6, 1 @ 1 invalidate D line\n\ stmia %0!, {r3, r4, ip, lr} @ 4\n\ @@ -38,7 +39,7 @@ static void v4wb_copy_user_page(void *kto, const void *kfrom) ldmia %1!, {r3, r4, ip, lr} @ 4\n\ subs %2, %2, #1 @ 1\n\ stmia %0!, {r3, r4, ip, lr} @ 4\n\ - ldmneia %1!, {r3, r4, ip, lr} @ 4\n\ + ldmiane %1!, {r3, r4, ip, lr} @ 4\n\ bne 1b @ 1\n\ mcr p15, 0, %1, c7, c10, 4 @ 1 drain WB" : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp) diff --git a/arch/arm/mm/copypage-v4wt.c b/arch/arm/mm/copypage-v4wt.c index 8614572e1296b..3851bb3964428 100644 --- a/arch/arm/mm/copypage-v4wt.c +++ b/arch/arm/mm/copypage-v4wt.c @@ -25,6 +25,7 @@ static void v4wt_copy_user_page(void *kto, const void *kfrom) int tmp; asm volatile ("\ + .syntax unified\n\ ldmia %1!, {r3, r4, ip, lr} @ 4\n\ 1: stmia %0!, {r3, r4, ip, lr} @ 4\n\ ldmia %1!, {r3, r4, ip, lr} @ 4+1\n\ @@ -34,7 +35,7 @@ static void v4wt_copy_user_page(void *kto, const void *kfrom) ldmia %1!, {r3, r4, ip, lr} @ 4\n\ subs %2, %2, #1 @ 1\n\ stmia %0!, {r3, r4, ip, lr} @ 4\n\ - ldmneia %1!, {r3, r4, ip, lr} @ 4\n\ + ldmiane %1!, {r3, r4, ip, lr} @ 4\n\ bne 1b @ 1\n\ mcr p15, 0, %2, c7, c7, 0 @ flush ID cache" : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index c6aab9c36ff18..43f46aa7ef335 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -2279,7 +2279,7 @@ EXPORT_SYMBOL_GPL(arm_iommu_attach_device); * @dev: valid struct device pointer * * Detaches the provided device from a previously attached map. - * This voids the dma operations (dma_map_ops pointer) + * This overwrites the dma_ops pointer with appropriate non-IOMMU ops. */ void arm_iommu_detach_device(struct device *dev) { diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c index 1d1edd0641995..a033f6134a649 100644 --- a/arch/arm/mm/idmap.c +++ b/arch/arm/mm/idmap.c @@ -6,6 +6,7 @@ #include <asm/cputype.h> #include <asm/idmap.h> +#include <asm/hwcap.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> #include <asm/sections.h> @@ -110,7 +111,8 @@ static int __init init_static_idmap(void) __idmap_text_end, 0); /* Flush L1 for the hardware to see this page table content */ - flush_cache_louis(); + if (!(elf_hwcap & HWCAP_LPAE)) + flush_cache_louis(); return 0; } diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 15dddfe433195..c2daabbe0af05 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -282,15 +282,12 @@ void __init arm_memblock_init(const struct machine_desc *mdesc) void __init bootmem_init(void) { - unsigned long min, max_low, max_high; - memblock_allow_resize(); - max_low = max_high = 0; - find_limits(&min, &max_low, &max_high); + find_limits(&min_low_pfn, &max_low_pfn, &max_pfn); - early_memtest((phys_addr_t)min << PAGE_SHIFT, - (phys_addr_t)max_low << PAGE_SHIFT); + early_memtest((phys_addr_t)min_low_pfn << PAGE_SHIFT, + (phys_addr_t)max_low_pfn << PAGE_SHIFT); /* * Sparsemem tries to allocate bootmem in memory_present(), @@ -308,16 +305,7 @@ void __init bootmem_init(void) * the sparse mem_map arrays initialized by sparse_init() * for memmap_init_zone(), otherwise all PFNs are invalid. */ - zone_sizes_init(min, max_low, max_high); - - /* - * This doesn't seem to be used by the Linux memory manager any - * more, but is used by ll_rw_block. If we can get rid of it, we - * also get rid of some of the stuff above as well. - */ - min_low_pfn = min; - max_low_pfn = max_low; - max_pfn = max_high; + zone_sizes_init(min_low_pfn, max_low_pfn, max_pfn); } /* @@ -498,55 +486,6 @@ void __init mem_init(void) mem_init_print_info(NULL); -#define MLK(b, t) b, t, ((t) - (b)) >> 10 -#define MLM(b, t) b, t, ((t) - (b)) >> 20 -#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K) - - pr_notice("Virtual kernel memory layout:\n" - " vector : 0x%08lx - 0x%08lx (%4ld kB)\n" -#ifdef CONFIG_HAVE_TCM - " DTCM : 0x%08lx - 0x%08lx (%4ld kB)\n" - " ITCM : 0x%08lx - 0x%08lx (%4ld kB)\n" -#endif - " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n" - " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n" - " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n" -#ifdef CONFIG_HIGHMEM - " pkmap : 0x%08lx - 0x%08lx (%4ld MB)\n" -#endif -#ifdef CONFIG_MODULES - " modules : 0x%08lx - 0x%08lx (%4ld MB)\n" -#endif - " .text : 0x%p" " - 0x%p" " (%4td kB)\n" - " .init : 0x%p" " - 0x%p" " (%4td kB)\n" - " .data : 0x%p" " - 0x%p" " (%4td kB)\n" - " .bss : 0x%p" " - 0x%p" " (%4td kB)\n", - - MLK(VECTORS_BASE, VECTORS_BASE + PAGE_SIZE), -#ifdef CONFIG_HAVE_TCM - MLK(DTCM_OFFSET, (unsigned long) dtcm_end), - MLK(ITCM_OFFSET, (unsigned long) itcm_end), -#endif - MLK(FIXADDR_START, FIXADDR_END), - MLM(VMALLOC_START, VMALLOC_END), - MLM(PAGE_OFFSET, (unsigned long)high_memory), -#ifdef CONFIG_HIGHMEM - MLM(PKMAP_BASE, (PKMAP_BASE) + (LAST_PKMAP) * - (PAGE_SIZE)), -#endif -#ifdef CONFIG_MODULES - MLM(MODULES_VADDR, MODULES_END), -#endif - - MLK_ROUNDUP(_text, _etext), - MLK_ROUNDUP(__init_begin, __init_end), - MLK_ROUNDUP(_sdata, _edata), - MLK_ROUNDUP(__bss_start, __bss_stop)); - -#undef MLK -#undef MLM -#undef MLK_ROUNDUP - /* * Check boundaries twice: Some fundamental inconsistencies can * be detected at build time already. diff --git a/arch/arm/mm/pmsa-v8.c b/arch/arm/mm/pmsa-v8.c index 617a83def88a9..0d7d5fb59247d 100644 --- a/arch/arm/mm/pmsa-v8.c +++ b/arch/arm/mm/pmsa-v8.c @@ -165,7 +165,7 @@ static int __init pmsav8_setup_ram(unsigned int number, phys_addr_t start,phys_a return -EINVAL; bar = start; - lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);; + lar = (end - 1) & ~(PMSAv8_MINALIGN - 1); bar |= PMSAv8_AP_PL1RW_PL0RW | PMSAv8_RGN_SHARED; lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN; @@ -181,7 +181,7 @@ static int __init pmsav8_setup_io(unsigned int number, phys_addr_t start,phys_ad return -EINVAL; bar = start; - lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);; + lar = (end - 1) & ~(PMSAv8_MINALIGN - 1); bar |= PMSAv8_AP_PL1RW_PL0RW | PMSAv8_RGN_SHARED | PMSAv8_BAR_XN; lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_DEVICE_nGnRnE) | PMSAv8_LAR_EN; diff --git a/arch/arm/mm/proc-v7m.S b/arch/arm/mm/proc-v7m.S index 47a5acc644333..acd5a66dfc23b 100644 --- a/arch/arm/mm/proc-v7m.S +++ b/arch/arm/mm/proc-v7m.S @@ -139,6 +139,9 @@ __v7m_setup_cont: cpsie i svc #0 1: cpsid i + ldr r0, =exc_ret + orr lr, lr, #EXC_RET_THREADMODE_PROCESSSTACK + str lr, [r0] ldmia sp, {r0-r3, r12} str r5, [r12, #11 * 4] @ restore the original SVC vector entry mov lr, r6 @ restore LR @@ -149,10 +152,10 @@ __v7m_setup_cont: @ Configure caches (if implemented) teq r8, #0 - stmneia sp, {r0-r6, lr} @ v7m_invalidate_l1 touches r0-r6 + stmiane sp, {r0-r6, lr} @ v7m_invalidate_l1 touches r0-r6 blne v7m_invalidate_l1 teq r8, #0 @ re-evalutae condition - ldmneia sp, {r0-r6, lr} + ldmiane sp, {r0-r6, lr} @ Configure the System Control Register to ensure 8-byte stack alignment @ Note the STKALIGN bit is either RW or RAO. diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 506386a3eddec..d3842791e1c42 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -77,6 +77,10 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) */ if (!vcpu_el1_is_32bit(vcpu)) vcpu->arch.hcr_el2 |= HCR_TID3; + + if (cpus_have_const_cap(ARM64_MISMATCHED_CACHE_TYPE) || + vcpu_el1_is_32bit(vcpu)) + vcpu->arch.hcr_el2 |= HCR_TID2; } static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) @@ -331,6 +335,14 @@ static inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu) return ESR_ELx_SYS64_ISS_RT(esr); } +static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) +{ + if (kvm_vcpu_trap_is_iabt(vcpu)) + return false; + + return kvm_vcpu_dabt_iswrite(vcpu); +} + static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) { return vcpu_read_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK; diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 222af1d2c3e4a..a01fe087e0228 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -31,6 +31,7 @@ #include <asm/kvm.h> #include <asm/kvm_asm.h> #include <asm/kvm_mmio.h> +#include <asm/smp_plat.h> #include <asm/thread_info.h> #define __KVM_HAVE_ARCH_INTC_INITIALIZED @@ -58,16 +59,19 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu); int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext); void __extended_idmap_trampoline(phys_addr_t boot_pgd, phys_addr_t idmap_start); -struct kvm_arch { +struct kvm_vmid { /* The VMID generation used for the virt. memory system */ u64 vmid_gen; u32 vmid; +}; + +struct kvm_arch { + struct kvm_vmid vmid; /* stage2 entry level table */ pgd_t *pgd; + phys_addr_t pgd_phys; - /* VTTBR value associated with above pgd and vmid */ - u64 vttbr; /* VTCR_EL2 value for this VM */ u64 vtcr; @@ -382,7 +386,36 @@ void kvm_arm_halt_guest(struct kvm *kvm); void kvm_arm_resume_guest(struct kvm *kvm); u64 __kvm_call_hyp(void *hypfn, ...); -#define kvm_call_hyp(f, ...) __kvm_call_hyp(kvm_ksym_ref(f), ##__VA_ARGS__) + +/* + * The couple of isb() below are there to guarantee the same behaviour + * on VHE as on !VHE, where the eret to EL1 acts as a context + * synchronization event. + */ +#define kvm_call_hyp(f, ...) \ + do { \ + if (has_vhe()) { \ + f(__VA_ARGS__); \ + isb(); \ + } else { \ + __kvm_call_hyp(kvm_ksym_ref(f), ##__VA_ARGS__); \ + } \ + } while(0) + +#define kvm_call_hyp_ret(f, ...) \ + ({ \ + typeof(f(__VA_ARGS__)) ret; \ + \ + if (has_vhe()) { \ + ret = f(__VA_ARGS__); \ + isb(); \ + } else { \ + ret = __kvm_call_hyp(kvm_ksym_ref(f), \ + ##__VA_ARGS__); \ + } \ + \ + ret; \ + }) void force_vm_exit(const cpumask_t *mask); void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot); @@ -401,6 +434,13 @@ struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr); DECLARE_PER_CPU(kvm_cpu_context_t, kvm_host_cpu_state); +static inline void kvm_init_host_cpu_context(kvm_cpu_context_t *cpu_ctxt, + int cpu) +{ + /* The host's MPIDR is immutable, so let's set it up at boot time */ + cpu_ctxt->sys_regs[MPIDR_EL1] = cpu_logical_map(cpu); +} + void __kvm_enable_ssbs(void); static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr, diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index a80a7ef573252..4da765f2cca58 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -21,6 +21,7 @@ #include <linux/compiler.h> #include <linux/kvm_host.h> #include <asm/alternative.h> +#include <asm/kvm_mmu.h> #include <asm/sysreg.h> #define __hyp_text __section(.hyp.text) notrace @@ -163,7 +164,7 @@ void __noreturn __hyp_do_panic(unsigned long, ...); static __always_inline void __hyp_text __load_guest_stage2(struct kvm *kvm) { write_sysreg(kvm->arch.vtcr, vtcr_el2); - write_sysreg(kvm->arch.vttbr, vttbr_el2); + write_sysreg(kvm_get_vttbr(kvm), vttbr_el2); /* * ARM erratum 1165522 requires the actual execution of the above diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 8af4b1befa421..b0742a16c6c9e 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -138,7 +138,8 @@ static inline unsigned long __kern_hyp_va(unsigned long v) }) /* - * We currently only support a 40bit IPA. + * We currently support using a VM-specified IPA size. For backward + * compatibility, the default IPA size is fixed to 40bits. */ #define KVM_PHYS_SHIFT (40) @@ -591,9 +592,15 @@ static inline u64 kvm_vttbr_baddr_mask(struct kvm *kvm) return vttbr_baddr_mask(kvm_phys_shift(kvm), kvm_stage2_levels(kvm)); } -static inline bool kvm_cpu_has_cnp(void) +static __always_inline u64 kvm_get_vttbr(struct kvm *kvm) { - return system_supports_cnp(); + struct kvm_vmid *vmid = &kvm->arch.vmid; + u64 vmid_field, baddr; + u64 cnp = system_supports_cnp() ? VTTBR_CNP_BIT : 0; + + baddr = kvm->arch.pgd_phys; + vmid_field = (u64)vmid->vmid << VTTBR_VMID_SHIFT; + return kvm_phys_to_vttbr(baddr) | vmid_field | cnp; } #endif /* __ASSEMBLY__ */ diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 72dc4c011014c..5b267dec6194e 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -361,6 +361,7 @@ #define SYS_CNTKCTL_EL1 sys_reg(3, 0, 14, 1, 0) +#define SYS_CCSIDR_EL1 sys_reg(3, 1, 0, 0, 0) #define SYS_CLIDR_EL1 sys_reg(3, 1, 0, 0, 1) #define SYS_AIDR_EL1 sys_reg(3, 1, 0, 0, 7) @@ -392,6 +393,10 @@ #define SYS_CNTP_CTL_EL0 sys_reg(3, 3, 14, 2, 1) #define SYS_CNTP_CVAL_EL0 sys_reg(3, 3, 14, 2, 2) +#define SYS_AARCH32_CNTP_TVAL sys_reg(0, 0, 14, 2, 0) +#define SYS_AARCH32_CNTP_CTL sys_reg(0, 0, 14, 2, 1) +#define SYS_AARCH32_CNTP_CVAL sys_reg(0, 2, 0, 14, 0) + #define __PMEV_op2(n) ((n) & 0x7) #define __CNTR_CRm(n) (0x8 | (((n) >> 3) & 0x3)) #define SYS_PMEVCNTRn_EL0(n) sys_reg(3, 3, 14, __CNTR_CRm(n), __PMEV_op2(n)) @@ -426,7 +431,7 @@ #define SYS_ICH_VTR_EL2 sys_reg(3, 4, 12, 11, 1) #define SYS_ICH_MISR_EL2 sys_reg(3, 4, 12, 11, 2) #define SYS_ICH_EISR_EL2 sys_reg(3, 4, 12, 11, 3) -#define SYS_ICH_ELSR_EL2 sys_reg(3, 4, 12, 11, 5) +#define SYS_ICH_ELRSR_EL2 sys_reg(3, 4, 12, 11, 5) #define SYS_ICH_VMCR_EL2 sys_reg(3, 4, 12, 11, 7) #define __SYS__LR0_EL2(x) sys_reg(3, 4, 12, 12, x) diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 0f2a135ba15bb..690e033a91c00 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -3,9 +3,7 @@ # Makefile for Kernel-based Virtual Machine module # -ccflags-y += -Iarch/arm64/kvm -Ivirt/kvm/arm/vgic -CFLAGS_arm.o := -I. -CFLAGS_mmu.o := -I. +ccflags-y += -I $(srctree)/$(src) -I $(srctree)/virt/kvm/arm/vgic KVM=../../../virt/kvm diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c index f39801e4136cd..fd917d6d12afb 100644 --- a/arch/arm64/kvm/debug.c +++ b/arch/arm64/kvm/debug.c @@ -76,7 +76,7 @@ static void restore_guest_debug_regs(struct kvm_vcpu *vcpu) void kvm_arm_init_debug(void) { - __this_cpu_write(mdcr_el2, kvm_call_hyp(__kvm_get_mdcr_el2)); + __this_cpu_write(mdcr_el2, kvm_call_hyp_ret(__kvm_get_mdcr_el2)); } /** diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S index 952f6cb9cf720..2845aa680841e 100644 --- a/arch/arm64/kvm/hyp.S +++ b/arch/arm64/kvm/hyp.S @@ -40,9 +40,6 @@ * arch/arm64/kernel/hyp_stub.S. */ ENTRY(__kvm_call_hyp) -alternative_if_not ARM64_HAS_VIRT_HOST_EXTN hvc #0 ret -alternative_else_nop_endif - b __vhe_hyp_call ENDPROC(__kvm_call_hyp) diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index 73c1b483ec396..2b1e686772bfd 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -43,18 +43,6 @@ ldr lr, [sp], #16 .endm -ENTRY(__vhe_hyp_call) - do_el2_call - /* - * We used to rely on having an exception return to get - * an implicit isb. In the E2H case, we don't have it anymore. - * rather than changing all the leaf functions, just do it here - * before returning to the rest of the kernel. - */ - isb - ret -ENDPROC(__vhe_hyp_call) - el1_sync: // Guest trapped into EL2 mrs x0, esr_el2 diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c index b426e2cf973cf..c52a8451637c4 100644 --- a/arch/arm64/kvm/hyp/sysreg-sr.c +++ b/arch/arm64/kvm/hyp/sysreg-sr.c @@ -53,7 +53,6 @@ static void __hyp_text __sysreg_save_user_state(struct kvm_cpu_context *ctxt) static void __hyp_text __sysreg_save_el1_state(struct kvm_cpu_context *ctxt) { - ctxt->sys_regs[MPIDR_EL1] = read_sysreg(vmpidr_el2); ctxt->sys_regs[CSSELR_EL1] = read_sysreg(csselr_el1); ctxt->sys_regs[SCTLR_EL1] = read_sysreg_el1(sctlr); ctxt->sys_regs[ACTLR_EL1] = read_sysreg(actlr_el1); diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index c936aa40c3f4a..539feecda5b81 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -982,6 +982,10 @@ static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, return true; } +#define reg_to_encoding(x) \ + sys_reg((u32)(x)->Op0, (u32)(x)->Op1, \ + (u32)(x)->CRn, (u32)(x)->CRm, (u32)(x)->Op2); + /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */ #define DBG_BCR_BVR_WCR_WVR_EL1(n) \ { SYS_DESC(SYS_DBGBVRn_EL1(n)), \ @@ -1003,44 +1007,38 @@ static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, { SYS_DESC(SYS_PMEVTYPERn_EL0(n)), \ access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), } -static bool access_cntp_tval(struct kvm_vcpu *vcpu, - struct sys_reg_params *p, - const struct sys_reg_desc *r) +static bool access_arch_timer(struct kvm_vcpu *vcpu, + struct sys_reg_params *p, + const struct sys_reg_desc *r) { - u64 now = kvm_phys_timer_read(); - u64 cval; + enum kvm_arch_timers tmr; + enum kvm_arch_timer_regs treg; + u64 reg = reg_to_encoding(r); - if (p->is_write) { - kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL, - p->regval + now); - } else { - cval = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL); - p->regval = cval - now; + switch (reg) { + case SYS_CNTP_TVAL_EL0: + case SYS_AARCH32_CNTP_TVAL: + tmr = TIMER_PTIMER; + treg = TIMER_REG_TVAL; + break; + case SYS_CNTP_CTL_EL0: + case SYS_AARCH32_CNTP_CTL: + tmr = TIMER_PTIMER; + treg = TIMER_REG_CTL; + break; + case SYS_CNTP_CVAL_EL0: + case SYS_AARCH32_CNTP_CVAL: + tmr = TIMER_PTIMER; + treg = TIMER_REG_CVAL; + break; + default: + BUG(); } - return true; -} - -static bool access_cntp_ctl(struct kvm_vcpu *vcpu, - struct sys_reg_params *p, - const struct sys_reg_desc *r) -{ - if (p->is_write) - kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CTL, p->regval); - else - p->regval = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CTL); - - return true; -} - -static bool access_cntp_cval(struct kvm_vcpu *vcpu, - struct sys_reg_params *p, - const struct sys_reg_desc *r) -{ if (p->is_write) - kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL, p->regval); + kvm_arm_timer_write_sysreg(vcpu, tmr, treg, p->regval); else - p->regval = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL); + p->regval = kvm_arm_timer_read_sysreg(vcpu, tmr, treg); return true; } @@ -1160,6 +1158,64 @@ static int set_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, return __set_id_reg(rd, uaddr, true); } +static bool access_ctr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + if (p->is_write) + return write_to_read_only(vcpu, p, r); + + p->regval = read_sanitised_ftr_reg(SYS_CTR_EL0); + return true; +} + +static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + if (p->is_write) + return write_to_read_only(vcpu, p, r); + + p->regval = read_sysreg(clidr_el1); + return true; +} + +static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + if (p->is_write) + vcpu_write_sys_reg(vcpu, p->regval, r->reg); + else + p->regval = vcpu_read_sys_reg(vcpu, r->reg); + return true; +} + +static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + u32 csselr; + + if (p->is_write) + return write_to_read_only(vcpu, p, r); + + csselr = vcpu_read_sys_reg(vcpu, CSSELR_EL1); + p->regval = get_ccsidr(csselr); + + /* + * Guests should not be doing cache operations by set/way at all, and + * for this reason, we trap them and attempt to infer the intent, so + * that we can flush the entire guest's address space at the appropriate + * time. + * To prevent this trapping from causing performance problems, let's + * expose the geometry of all data and unified caches (which are + * guaranteed to be PIPT and thus non-aliasing) as 1 set and 1 way. + * [If guests should attempt to infer aliasing properties from the + * geometry (which is not permitted by the architecture), they would + * only do so for virtually indexed caches.] + */ + if (!(csselr & 1)) // data or unified cache + p->regval &= ~GENMASK(27, 3); + return true; +} + /* sys_reg_desc initialiser for known cpufeature ID registers */ #define ID_SANITISED(name) { \ SYS_DESC(SYS_##name), \ @@ -1377,7 +1433,10 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0}, - { SYS_DESC(SYS_CSSELR_EL1), NULL, reset_unknown, CSSELR_EL1 }, + { SYS_DESC(SYS_CCSIDR_EL1), access_ccsidr }, + { SYS_DESC(SYS_CLIDR_EL1), access_clidr }, + { SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 }, + { SYS_DESC(SYS_CTR_EL0), access_ctr }, { SYS_DESC(SYS_PMCR_EL0), access_pmcr, reset_pmcr, }, { SYS_DESC(SYS_PMCNTENSET_EL0), access_pmcnten, reset_unknown, PMCNTENSET_EL0 }, @@ -1400,9 +1459,9 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 }, { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 }, - { SYS_DESC(SYS_CNTP_TVAL_EL0), access_cntp_tval }, - { SYS_DESC(SYS_CNTP_CTL_EL0), access_cntp_ctl }, - { SYS_DESC(SYS_CNTP_CVAL_EL0), access_cntp_cval }, + { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer }, + { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer }, + { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer }, /* PMEVCNTRn_EL0 */ PMU_PMEVCNTR_EL0(0), @@ -1476,7 +1535,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_DACR32_EL2), NULL, reset_unknown, DACR32_EL2 }, { SYS_DESC(SYS_IFSR32_EL2), NULL, reset_unknown, IFSR32_EL2 }, - { SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x70 }, + { SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x700 }, }; static bool trap_dbgidr(struct kvm_vcpu *vcpu, @@ -1677,6 +1736,7 @@ static const struct sys_reg_desc cp14_64_regs[] = { * register). */ static const struct sys_reg_desc cp15_regs[] = { + { Op1( 0), CRn( 0), CRm( 0), Op2( 1), access_ctr }, { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR }, { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 }, { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 }, @@ -1723,10 +1783,9 @@ static const struct sys_reg_desc cp15_regs[] = { { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, c13_CID }, - /* CNTP_TVAL */ - { Op1( 0), CRn(14), CRm( 2), Op2( 0), access_cntp_tval }, - /* CNTP_CTL */ - { Op1( 0), CRn(14), CRm( 2), Op2( 1), access_cntp_ctl }, + /* Arch Tmers */ + { SYS_DESC(SYS_AARCH32_CNTP_TVAL), access_arch_timer }, + { SYS_DESC(SYS_AARCH32_CNTP_CTL), access_arch_timer }, /* PMEVCNTRn */ PMU_PMEVCNTR(0), @@ -1794,6 +1853,10 @@ static const struct sys_reg_desc cp15_regs[] = { PMU_PMEVTYPER(30), /* PMCCFILTR */ { Op1(0), CRn(14), CRm(15), Op2(7), access_pmu_evtyper }, + + { Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr }, + { Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr }, + { Op1(2), CRn( 0), CRm( 0), Op2(0), access_csselr, NULL, c0_CSSELR }, }; static const struct sys_reg_desc cp15_64_regs[] = { @@ -1803,7 +1866,7 @@ static const struct sys_reg_desc cp15_64_regs[] = { { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 }, { Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_ASGI1R */ { Op1( 2), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI0R */ - { Op1( 2), CRn( 0), CRm(14), Op2( 0), access_cntp_cval }, + { SYS_DESC(SYS_AARCH32_CNTP_CVAL), access_arch_timer }, }; /* Target specific emulation tables */ @@ -1832,30 +1895,19 @@ static const struct sys_reg_desc *get_target_table(unsigned target, } } -#define reg_to_match_value(x) \ - ({ \ - unsigned long val; \ - val = (x)->Op0 << 14; \ - val |= (x)->Op1 << 11; \ - val |= (x)->CRn << 7; \ - val |= (x)->CRm << 3; \ - val |= (x)->Op2; \ - val; \ - }) - static int match_sys_reg(const void *key, const void *elt) { const unsigned long pval = (unsigned long)key; const struct sys_reg_desc *r = elt; - return pval - reg_to_match_value(r); + return pval - reg_to_encoding(r); } static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params, const struct sys_reg_desc table[], unsigned int num) { - unsigned long pval = reg_to_match_value(params); + unsigned long pval = reg_to_encoding(params); return bsearch((void *)pval, table, num, sizeof(table[0]), match_sys_reg); } @@ -2218,11 +2270,15 @@ static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu, } FUNCTION_INVARIANT(midr_el1) -FUNCTION_INVARIANT(ctr_el0) FUNCTION_INVARIANT(revidr_el1) FUNCTION_INVARIANT(clidr_el1) FUNCTION_INVARIANT(aidr_el1) +static void get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r) +{ + ((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0); +} + /* ->val is filled in by kvm_sys_reg_table_init() */ static struct sys_reg_desc invariant_sys_regs[] = { { SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 }, diff --git a/arch/ia64/hp/sim/simscsi.c b/arch/ia64/hp/sim/simscsi.c index f86844fc0725b..0a8a742711730 100644 --- a/arch/ia64/hp/sim/simscsi.c +++ b/arch/ia64/hp/sim/simscsi.c @@ -105,7 +105,8 @@ simscsi_interrupt (unsigned long val) atomic_dec(&num_reqs); queue[rd].sc = NULL; if (DBG) - printk("simscsi_interrupt: done with %ld\n", sc->serial_number); + printk("simscsi_interrupt: done with %u\n", + sc->request->tag); (*sc->scsi_done)(sc); rd = (rd + 1) % SIMSCSI_REQ_QUEUE_LEN; } @@ -214,8 +215,8 @@ simscsi_queuecommand_lck (struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *) register long sp asm ("sp"); if (DBG) - printk("simscsi_queuecommand: target=%d,cmnd=%u,sc=%lu,sp=%lx,done=%p\n", - target_id, sc->cmnd[0], sc->serial_number, sp, done); + printk("simscsi_queuecommand: target=%d,cmnd=%u,sc=%u,sp=%lx,done=%p\n", + target_id, sc->cmnd[0], sc->request->tag, sp, done); #endif sc->result = DID_BAD_TARGET << 16; diff --git a/arch/microblaze/kernel/setup.c b/arch/microblaze/kernel/setup.c index bbd6968ce55b5..522a0c5d9c59c 100644 --- a/arch/microblaze/kernel/setup.c +++ b/arch/microblaze/kernel/setup.c @@ -192,23 +192,14 @@ struct dentry *of_debugfs_root; static int microblaze_debugfs_init(void) { of_debugfs_root = debugfs_create_dir("microblaze", NULL); - - return of_debugfs_root == NULL; + return 0; } arch_initcall(microblaze_debugfs_init); # ifdef CONFIG_MMU static int __init debugfs_tlb(void) { - struct dentry *d; - - if (!of_debugfs_root) - return -ENODEV; - - d = debugfs_create_u32("tlb_skip", S_IRUGO, of_debugfs_root, &tlb_skip); - if (!d) - return -ENOMEM; - + debugfs_create_u32("tlb_skip", S_IRUGO, of_debugfs_root, &tlb_skip); return 0; } device_initcall(debugfs_tlb); diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index d2abd98471e86..41204a49cf95e 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h @@ -1134,7 +1134,7 @@ static inline void kvm_arch_hardware_unsetup(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, struct kvm_memory_slot *dont) {} -static inline void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} diff --git a/arch/mips/include/uapi/asm/socket.h b/arch/mips/include/uapi/asm/socket.h index eb9f33f8a8b37..d41765cfbc6eb 100644 --- a/arch/mips/include/uapi/asm/socket.h +++ b/arch/mips/include/uapi/asm/socket.h @@ -10,8 +10,8 @@ #ifndef _UAPI_ASM_SOCKET_H #define _UAPI_ASM_SOCKET_H +#include <linux/posix_types.h> #include <asm/sockios.h> -#include <asm/bitsperlong.h> /* * For setsockopt(2) diff --git a/arch/parisc/include/uapi/asm/socket.h b/arch/parisc/include/uapi/asm/socket.h index 16e428f03526a..66c5dd245ac75 100644 --- a/arch/parisc/include/uapi/asm/socket.h +++ b/arch/parisc/include/uapi/asm/socket.h @@ -2,8 +2,8 @@ #ifndef _UAPI_ASM_SOCKET_H #define _UAPI_ASM_SOCKET_H +#include <linux/posix_types.h> #include <asm/sockios.h> -#include <asm/bitsperlong.h> /* For setsockopt(2) */ #define SOL_SOCKET 0xffff diff --git a/arch/powerpc/configs/skiroot_defconfig b/arch/powerpc/configs/skiroot_defconfig index cfdd08897a060..5ba131c30f6bc 100644 --- a/arch/powerpc/configs/skiroot_defconfig +++ b/arch/powerpc/configs/skiroot_defconfig @@ -37,7 +37,8 @@ CONFIG_MODULE_SIG=y CONFIG_MODULE_SIG_FORCE=y CONFIG_MODULE_SIG_SHA512=y CONFIG_PARTITION_ADVANCED=y -# CONFIG_IOSCHED_DEADLINE is not set +# CONFIG_MQ_IOSCHED_DEADLINE is not set +# CONFIG_MQ_IOSCHED_KYBER is not set # CONFIG_PPC_VAS is not set # CONFIG_PPC_PSERIES is not set # CONFIG_PPC_OF_BOOT_TRAMPOLINE is not set @@ -49,7 +50,6 @@ CONFIG_IRQ_ALL_CPUS=y CONFIG_NUMA=y # CONFIG_COMPACTION is not set # CONFIG_MIGRATION is not set -# CONFIG_BOUNCE is not set CONFIG_PPC_64K_PAGES=y CONFIG_SCHED_SMT=y CONFIG_CMDLINE_BOOL=y @@ -136,9 +136,11 @@ CONFIG_ACENIC_OMIT_TIGON_I=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set # CONFIG_NET_VENDOR_ATHEROS is not set +# CONFIG_NET_VENDOR_AURORA is not set CONFIG_TIGON3=m CONFIG_BNX2X=m # CONFIG_NET_VENDOR_BROCADE is not set +# CONFIG_NET_VENDOR_CADENCE is not set # CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_CAVIUM is not set CONFIG_CHELSIO_T1=m @@ -151,6 +153,7 @@ CONFIG_BE2NET=m # CONFIG_NET_VENDOR_HP is not set # CONFIG_NET_VENDOR_HUAWEI is not set CONFIG_E1000=m +CONFIG_E1000E=m CONFIG_IGB=m CONFIG_IXGB=m CONFIG_IXGBE=m @@ -161,15 +164,18 @@ CONFIG_MLX4_EN=m # CONFIG_MLX4_CORE_GEN2 is not set CONFIG_MLX5_CORE=m # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set CONFIG_MYRI10GE=m # CONFIG_NET_VENDOR_NATSEMI is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set # CONFIG_NET_VENDOR_NVIDIA is not set # CONFIG_NET_VENDOR_OKI is not set -# CONFIG_NET_PACKET_ENGINE is not set +# CONFIG_NET_VENDOR_PACKET_ENGINES is not set CONFIG_QLGE=m CONFIG_NETXEN_NIC=m +CONFIG_QED=m +CONFIG_QEDE=m # CONFIG_NET_VENDOR_QUALCOMM is not set # CONFIG_NET_VENDOR_RDC is not set # CONFIG_NET_VENDOR_REALTEK is not set diff --git a/arch/powerpc/include/asm/book3s/64/hugetlb.h b/arch/powerpc/include/asm/book3s/64/hugetlb.h index 66c1e4f88d654..ec2a55a553c75 100644 --- a/arch/powerpc/include/asm/book3s/64/hugetlb.h +++ b/arch/powerpc/include/asm/book3s/64/hugetlb.h @@ -39,6 +39,14 @@ static inline int hstate_get_psize(struct hstate *hstate) #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE static inline bool gigantic_page_supported(void) { + /* + * We used gigantic page reservation with hypervisor assist in some case. + * We cannot use runtime allocation of gigantic pages in those platforms + * This is hash translation mode LPARs. + */ + if (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled()) + return false; + return true; } #endif diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 0f98f00da2ea3..e6b5bb012ccb9 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -99,6 +99,8 @@ struct kvm_nested_guest; struct kvm_vm_stat { ulong remote_tlb_flush; + ulong num_2M_pages; + ulong num_1G_pages; }; struct kvm_vcpu_stat { @@ -377,6 +379,7 @@ struct kvmppc_mmu { void (*slbmte)(struct kvm_vcpu *vcpu, u64 rb, u64 rs); u64 (*slbmfee)(struct kvm_vcpu *vcpu, u64 slb_nr); u64 (*slbmfev)(struct kvm_vcpu *vcpu, u64 slb_nr); + int (*slbfee)(struct kvm_vcpu *vcpu, gva_t eaddr, ulong *ret_slb); void (*slbie)(struct kvm_vcpu *vcpu, u64 slb_nr); void (*slbia)(struct kvm_vcpu *vcpu); /* book3s */ @@ -837,7 +840,7 @@ struct kvm_vcpu_arch { static inline void kvm_arch_hardware_disable(void) {} static inline void kvm_arch_hardware_unsetup(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} -static inline void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_exit(void) {} diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index a6c8548ed9faa..ac22b28ae78d4 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -36,6 +36,8 @@ #endif #ifdef CONFIG_KVM_BOOK3S_64_HANDLER #include <asm/paca.h> +#include <asm/xive.h> +#include <asm/cpu_has_feature.h> #endif /* @@ -617,6 +619,18 @@ static inline int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 ir static inline void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu) { } #endif /* CONFIG_KVM_XIVE */ +#if defined(CONFIG_PPC_POWERNV) && defined(CONFIG_KVM_BOOK3S_64_HANDLER) +static inline bool xics_on_xive(void) +{ + return xive_enabled() && cpu_has_feature(CPU_FTR_HVMODE); +} +#else +static inline bool xics_on_xive(void) +{ + return false; +} +#endif + /* * Prototypes for functions called only from assembler code. * Having prototypes reduces sparse errors. diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h index 8c876c166ef27..26ca425f4c2c3 100644 --- a/arch/powerpc/include/uapi/asm/kvm.h +++ b/arch/powerpc/include/uapi/asm/kvm.h @@ -463,10 +463,12 @@ struct kvm_ppc_cpu_char { #define KVM_PPC_CPU_CHAR_BR_HINT_HONOURED (1ULL << 58) #define KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF (1ULL << 57) #define KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS (1ULL << 56) +#define KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST (1ull << 54) #define KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY (1ULL << 63) #define KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR (1ULL << 62) #define KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR (1ULL << 61) +#define KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE (1ull << 58) /* Per-vcpu XICS interrupt controller state */ #define KVM_REG_PPC_ICP_STATE (KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x8c) diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S index 1881127682e99..32332e24e4216 100644 --- a/arch/powerpc/kernel/head_fsl_booke.S +++ b/arch/powerpc/kernel/head_fsl_booke.S @@ -194,13 +194,6 @@ set_ivor: #endif mtspr SPRN_MAS4, r2 -#if 0 - /* Enable DOZE */ - mfspr r2,SPRN_HID0 - oris r2,r2,HID0_DOZE@h - mtspr SPRN_HID0, r2 -#endif - #if !defined(CONFIG_BDI_SWITCH) /* * The Abatron BDI JTAG debugger does not tolerate others diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index a21200c6aaeaa..1fd45a8650e17 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -71,6 +71,7 @@ #include <sysdev/fsl_pci.h> #include <asm/kprobes.h> #include <asm/stacktrace.h> +#include <asm/nmi.h> #if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC_CORE) int (*__debugger)(struct pt_regs *regs) __read_mostly; diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index 9a7dadbe1f173..10c5579d20cec 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -39,6 +39,7 @@ #include "book3s.h" #include "trace.h" +#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU /* #define EXIT_DEBUG */ @@ -71,6 +72,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "pthru_all", VCPU_STAT(pthru_all) }, { "pthru_host", VCPU_STAT(pthru_host) }, { "pthru_bad_aff", VCPU_STAT(pthru_bad_aff) }, + { "largepages_2M", VM_STAT(num_2M_pages) }, + { "largepages_1G", VM_STAT(num_1G_pages) }, { NULL } }; @@ -642,7 +645,7 @@ int kvmppc_get_one_reg(struct kvm_vcpu *vcpu, u64 id, r = -ENXIO; break; } - if (xive_enabled()) + if (xics_on_xive()) *val = get_reg_val(id, kvmppc_xive_get_icp(vcpu)); else *val = get_reg_val(id, kvmppc_xics_get_icp(vcpu)); @@ -715,7 +718,7 @@ int kvmppc_set_one_reg(struct kvm_vcpu *vcpu, u64 id, r = -ENXIO; break; } - if (xive_enabled()) + if (xics_on_xive()) r = kvmppc_xive_set_icp(vcpu, set_reg_val(id, *val)); else r = kvmppc_xics_set_icp(vcpu, set_reg_val(id, *val)); @@ -991,7 +994,7 @@ int kvmppc_book3s_hcall_implemented(struct kvm *kvm, unsigned long hcall) int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, bool line_status) { - if (xive_enabled()) + if (xics_on_xive()) return kvmppc_xive_set_irq(kvm, irq_source_id, irq, level, line_status); else @@ -1044,7 +1047,7 @@ static int kvmppc_book3s_init(void) #ifdef CONFIG_KVM_XICS #ifdef CONFIG_KVM_XIVE - if (xive_enabled()) { + if (xics_on_xive()) { kvmppc_xive_init_module(); kvm_register_device_ops(&kvm_xive_ops, KVM_DEV_TYPE_XICS); } else @@ -1057,7 +1060,7 @@ static int kvmppc_book3s_init(void) static void kvmppc_book3s_exit(void) { #ifdef CONFIG_KVM_XICS - if (xive_enabled()) + if (xics_on_xive()) kvmppc_xive_exit_module(); #endif #ifdef CONFIG_KVM_BOOK3S_32_HANDLER diff --git a/arch/powerpc/kvm/book3s_32_mmu.c b/arch/powerpc/kvm/book3s_32_mmu.c index 612169988a3d8..6f789f674048a 100644 --- a/arch/powerpc/kvm/book3s_32_mmu.c +++ b/arch/powerpc/kvm/book3s_32_mmu.c @@ -425,6 +425,7 @@ void kvmppc_mmu_book3s_32_init(struct kvm_vcpu *vcpu) mmu->slbmte = NULL; mmu->slbmfee = NULL; mmu->slbmfev = NULL; + mmu->slbfee = NULL; mmu->slbie = NULL; mmu->slbia = NULL; } diff --git a/arch/powerpc/kvm/book3s_64_mmu.c b/arch/powerpc/kvm/book3s_64_mmu.c index c92dd25bed237..d4b967f0e8d4b 100644 --- a/arch/powerpc/kvm/book3s_64_mmu.c +++ b/arch/powerpc/kvm/book3s_64_mmu.c @@ -435,6 +435,19 @@ static void kvmppc_mmu_book3s_64_slbmte(struct kvm_vcpu *vcpu, u64 rs, u64 rb) kvmppc_mmu_map_segment(vcpu, esid << SID_SHIFT); } +static int kvmppc_mmu_book3s_64_slbfee(struct kvm_vcpu *vcpu, gva_t eaddr, + ulong *ret_slb) +{ + struct kvmppc_slb *slbe = kvmppc_mmu_book3s_64_find_slbe(vcpu, eaddr); + + if (slbe) { + *ret_slb = slbe->origv; + return 0; + } + *ret_slb = 0; + return -ENOENT; +} + static u64 kvmppc_mmu_book3s_64_slbmfee(struct kvm_vcpu *vcpu, u64 slb_nr) { struct kvmppc_slb *slbe; @@ -670,6 +683,7 @@ void kvmppc_mmu_book3s_64_init(struct kvm_vcpu *vcpu) mmu->slbmte = kvmppc_mmu_book3s_64_slbmte; mmu->slbmfee = kvmppc_mmu_book3s_64_slbmfee; mmu->slbmfev = kvmppc_mmu_book3s_64_slbmfev; + mmu->slbfee = kvmppc_mmu_book3s_64_slbfee; mmu->slbie = kvmppc_mmu_book3s_64_slbie; mmu->slbia = kvmppc_mmu_book3s_64_slbia; mmu->xlate = kvmppc_mmu_book3s_64_xlate; diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index bd2dcfbf00cdb..be7bc070eae5f 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -442,6 +442,24 @@ int kvmppc_hv_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 last_inst; /* + * Fast path - check if the guest physical address corresponds to a + * device on the FAST_MMIO_BUS, if so we can avoid loading the + * instruction all together, then we can just handle it and return. + */ + if (is_store) { + int idx, ret; + + idx = srcu_read_lock(&vcpu->kvm->srcu); + ret = kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, (gpa_t) gpa, 0, + NULL); + srcu_read_unlock(&vcpu->kvm->srcu, idx); + if (!ret) { + kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + 4); + return RESUME_GUEST; + } + } + + /* * If we fail, we just return to the guest and try executing it again. */ if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) != diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c index 1b821c6efdefb..f55ef071883f1 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_radix.c +++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c @@ -403,8 +403,13 @@ void kvmppc_unmap_pte(struct kvm *kvm, pte_t *pte, unsigned long gpa, if (!memslot) return; } - if (shift) + if (shift) { /* 1GB or 2MB page */ page_size = 1ul << shift; + if (shift == PMD_SHIFT) + kvm->stat.num_2M_pages--; + else if (shift == PUD_SHIFT) + kvm->stat.num_1G_pages--; + } gpa &= ~(page_size - 1); hpa = old & PTE_RPN_MASK; @@ -878,6 +883,14 @@ int kvmppc_book3s_instantiate_page(struct kvm_vcpu *vcpu, put_page(page); } + /* Increment number of large pages if we (successfully) inserted one */ + if (!ret) { + if (level == 1) + kvm->stat.num_2M_pages++; + else if (level == 2) + kvm->stat.num_1G_pages++; + } + return ret; } diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c index 532ab79734c7a..f02b049737109 100644 --- a/arch/powerpc/kvm/book3s_64_vio.c +++ b/arch/powerpc/kvm/book3s_64_vio.c @@ -133,7 +133,6 @@ extern void kvm_spapr_tce_release_iommu_group(struct kvm *kvm, continue; kref_put(&stit->kref, kvm_spapr_tce_liobn_put); - return; } } } @@ -338,14 +337,15 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm, } } + kvm_get_kvm(kvm); if (!ret) ret = anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops, stt, O_RDWR | O_CLOEXEC); - if (ret >= 0) { + if (ret >= 0) list_add_rcu(&stt->list, &kvm->arch.spapr_tce_tables); - kvm_get_kvm(kvm); - } + else + kvm_put_kvm(kvm); mutex_unlock(&kvm->lock); diff --git a/arch/powerpc/kvm/book3s_emulate.c b/arch/powerpc/kvm/book3s_emulate.c index 8c7e933e942e5..6ef7c5f00a49c 100644 --- a/arch/powerpc/kvm/book3s_emulate.c +++ b/arch/powerpc/kvm/book3s_emulate.c @@ -47,6 +47,7 @@ #define OP_31_XOP_SLBMFEV 851 #define OP_31_XOP_EIOIO 854 #define OP_31_XOP_SLBMFEE 915 +#define OP_31_XOP_SLBFEE 979 #define OP_31_XOP_TBEGIN 654 #define OP_31_XOP_TABORT 910 @@ -416,6 +417,23 @@ int kvmppc_core_emulate_op_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, vcpu->arch.mmu.slbia(vcpu); break; + case OP_31_XOP_SLBFEE: + if (!(inst & 1) || !vcpu->arch.mmu.slbfee) { + return EMULATE_FAIL; + } else { + ulong b, t; + ulong cr = kvmppc_get_cr(vcpu) & ~CR0_MASK; + + b = kvmppc_get_gpr(vcpu, rb); + if (!vcpu->arch.mmu.slbfee(vcpu, b, &t)) + cr |= 2 << CR0_SHIFT; + kvmppc_set_gpr(vcpu, rt, t); + /* copy XER[SO] bit to CR0[SO] */ + cr |= (vcpu->arch.regs.xer & 0x80000000) >> + (31 - CR0_SHIFT); + kvmppc_set_cr(vcpu, cr); + } + break; case OP_31_XOP_SLBMFEE: if (!vcpu->arch.mmu.slbmfee) { emulated = EMULATE_FAIL; diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index a3d5318f5d1e9..06964350b97a9 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -922,7 +922,7 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) case H_IPOLL: case H_XIRR_X: if (kvmppc_xics_enabled(vcpu)) { - if (xive_enabled()) { + if (xics_on_xive()) { ret = H_NOT_AVAILABLE; return RESUME_GUEST; } @@ -937,6 +937,7 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) ret = kvmppc_h_set_xdabr(vcpu, kvmppc_get_gpr(vcpu, 4), kvmppc_get_gpr(vcpu, 5)); break; +#ifdef CONFIG_SPAPR_TCE_IOMMU case H_GET_TCE: ret = kvmppc_h_get_tce(vcpu, kvmppc_get_gpr(vcpu, 4), kvmppc_get_gpr(vcpu, 5)); @@ -966,6 +967,7 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) if (ret == H_TOO_HARD) return RESUME_HOST; break; +#endif case H_RANDOM: if (!powernv_get_random_long(&vcpu->arch.regs.gpr[4])) ret = H_HARDWARE; @@ -1445,7 +1447,7 @@ static int kvmppc_handle_nested_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) case BOOK3S_INTERRUPT_HV_RM_HARD: vcpu->arch.trap = 0; r = RESUME_GUEST; - if (!xive_enabled()) + if (!xics_on_xive()) kvmppc_xics_rm_complete(vcpu, 0); break; default: @@ -3648,11 +3650,12 @@ static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc, static void grow_halt_poll_ns(struct kvmppc_vcore *vc) { - /* 10us base */ - if (vc->halt_poll_ns == 0 && halt_poll_ns_grow) - vc->halt_poll_ns = 10000; - else - vc->halt_poll_ns *= halt_poll_ns_grow; + if (!halt_poll_ns_grow) + return; + + vc->halt_poll_ns *= halt_poll_ns_grow; + if (vc->halt_poll_ns < halt_poll_ns_grow_start) + vc->halt_poll_ns = halt_poll_ns_grow_start; } static void shrink_halt_poll_ns(struct kvmppc_vcore *vc) @@ -3666,7 +3669,7 @@ static void shrink_halt_poll_ns(struct kvmppc_vcore *vc) #ifdef CONFIG_KVM_XICS static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu) { - if (!xive_enabled()) + if (!xics_on_xive()) return false; return vcpu->arch.irq_pending || vcpu->arch.xive_saved_state.pipr < vcpu->arch.xive_saved_state.cppr; @@ -4226,7 +4229,7 @@ static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); srcu_read_unlock(&kvm->srcu, srcu_idx); } else if (r == RESUME_PASSTHROUGH) { - if (WARN_ON(xive_enabled())) + if (WARN_ON(xics_on_xive())) r = H_SUCCESS; else r = kvmppc_xics_rm_complete(vcpu, 0); @@ -4750,7 +4753,7 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm) * If xive is enabled, we route 0x500 interrupts directly * to the guest. */ - if (xive_enabled()) + if (xics_on_xive()) lpcr |= LPCR_LPES; } @@ -4986,7 +4989,7 @@ static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi) if (i == pimap->n_mapped) pimap->n_mapped++; - if (xive_enabled()) + if (xics_on_xive()) rc = kvmppc_xive_set_mapped(kvm, guest_gsi, desc); else kvmppc_xics_set_mapped(kvm, guest_gsi, desc->irq_data.hwirq); @@ -5027,7 +5030,7 @@ static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi) return -ENODEV; } - if (xive_enabled()) + if (xics_on_xive()) rc = kvmppc_xive_clr_mapped(kvm, guest_gsi, pimap->mapped[i].desc); else kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq); @@ -5359,13 +5362,11 @@ static int kvm_init_subcore_bitmap(void) continue; sibling_subcore_state = - kmalloc_node(sizeof(struct sibling_subcore_state), + kzalloc_node(sizeof(struct sibling_subcore_state), GFP_KERNEL, node); if (!sibling_subcore_state) return -ENOMEM; - memset(sibling_subcore_state, 0, - sizeof(struct sibling_subcore_state)); for (j = 0; j < threads_per_core; j++) { int cpu = first_cpu + j; @@ -5406,7 +5407,7 @@ static int kvmppc_book3s_init_hv(void) * indirectly, via OPAL. */ #ifdef CONFIG_SMP - if (!xive_enabled() && !kvmhv_on_pseries() && + if (!xics_on_xive() && !kvmhv_on_pseries() && !local_paca->kvm_hstate.xics_phys) { struct device_node *np; diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index a71e2fc00a4e8..b0cf22477e879 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c @@ -257,7 +257,7 @@ void kvmhv_rm_send_ipi(int cpu) } /* We should never reach this */ - if (WARN_ON_ONCE(xive_enabled())) + if (WARN_ON_ONCE(xics_on_xive())) return; /* Else poke the target with an IPI */ @@ -577,7 +577,7 @@ unsigned long kvmppc_rm_h_xirr(struct kvm_vcpu *vcpu) { if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; - if (xive_enabled()) { + if (xics_on_xive()) { if (is_rm()) return xive_rm_h_xirr(vcpu); if (unlikely(!__xive_vm_h_xirr)) @@ -592,7 +592,7 @@ unsigned long kvmppc_rm_h_xirr_x(struct kvm_vcpu *vcpu) if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; vcpu->arch.regs.gpr[5] = get_tb(); - if (xive_enabled()) { + if (xics_on_xive()) { if (is_rm()) return xive_rm_h_xirr(vcpu); if (unlikely(!__xive_vm_h_xirr)) @@ -606,7 +606,7 @@ unsigned long kvmppc_rm_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server) { if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; - if (xive_enabled()) { + if (xics_on_xive()) { if (is_rm()) return xive_rm_h_ipoll(vcpu, server); if (unlikely(!__xive_vm_h_ipoll)) @@ -621,7 +621,7 @@ int kvmppc_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, { if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; - if (xive_enabled()) { + if (xics_on_xive()) { if (is_rm()) return xive_rm_h_ipi(vcpu, server, mfrr); if (unlikely(!__xive_vm_h_ipi)) @@ -635,7 +635,7 @@ int kvmppc_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr) { if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; - if (xive_enabled()) { + if (xics_on_xive()) { if (is_rm()) return xive_rm_h_cppr(vcpu, cppr); if (unlikely(!__xive_vm_h_cppr)) @@ -649,7 +649,7 @@ int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr) { if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; - if (xive_enabled()) { + if (xics_on_xive()) { if (is_rm()) return xive_rm_h_eoi(vcpu, xirr); if (unlikely(!__xive_vm_h_eoi)) diff --git a/arch/powerpc/kvm/book3s_hv_rm_xics.c b/arch/powerpc/kvm/book3s_hv_rm_xics.c index b3f5786b20dcf..3b9662a4207e0 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_xics.c +++ b/arch/powerpc/kvm/book3s_hv_rm_xics.c @@ -144,6 +144,13 @@ static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu, return; } + if (xive_enabled() && kvmhv_on_pseries()) { + /* No XICS access or hypercalls available, too hard */ + this_icp->rm_action |= XICS_RM_KICK_VCPU; + this_icp->rm_kick_target = vcpu; + return; + } + /* * Check if the core is loaded, * if not, find an available host core to post to wake the VCPU, diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index 25043b50cb30a..3a5e719ef032b 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -2272,8 +2272,13 @@ hcall_real_table: .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table .long DOTSYM(kvmppc_h_protect) - hcall_real_table +#ifdef CONFIG_SPAPR_TCE_IOMMU .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table +#else + .long 0 /* 0x1c */ + .long 0 /* 0x20 */ +#endif .long 0 /* 0x24 - H_SET_SPRG0 */ .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table .long 0 /* 0x2c */ @@ -2351,8 +2356,13 @@ hcall_real_table: .long 0 /* 0x12c */ .long 0 /* 0x130 */ .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table +#ifdef CONFIG_SPAPR_TCE_IOMMU .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table +#else + .long 0 /* 0x138 */ + .long 0 /* 0x13c */ +#endif .long 0 /* 0x140 */ .long 0 /* 0x144 */ .long 0 /* 0x148 */ diff --git a/arch/powerpc/kvm/book3s_rtas.c b/arch/powerpc/kvm/book3s_rtas.c index 2d3b2b1cc272b..4e178c4c1ea50 100644 --- a/arch/powerpc/kvm/book3s_rtas.c +++ b/arch/powerpc/kvm/book3s_rtas.c @@ -33,7 +33,7 @@ static void kvm_rtas_set_xive(struct kvm_vcpu *vcpu, struct rtas_args *args) server = be32_to_cpu(args->args[1]); priority = be32_to_cpu(args->args[2]); - if (xive_enabled()) + if (xics_on_xive()) rc = kvmppc_xive_set_xive(vcpu->kvm, irq, server, priority); else rc = kvmppc_xics_set_xive(vcpu->kvm, irq, server, priority); @@ -56,7 +56,7 @@ static void kvm_rtas_get_xive(struct kvm_vcpu *vcpu, struct rtas_args *args) irq = be32_to_cpu(args->args[0]); server = priority = 0; - if (xive_enabled()) + if (xics_on_xive()) rc = kvmppc_xive_get_xive(vcpu->kvm, irq, &server, &priority); else rc = kvmppc_xics_get_xive(vcpu->kvm, irq, &server, &priority); @@ -83,7 +83,7 @@ static void kvm_rtas_int_off(struct kvm_vcpu *vcpu, struct rtas_args *args) irq = be32_to_cpu(args->args[0]); - if (xive_enabled()) + if (xics_on_xive()) rc = kvmppc_xive_int_off(vcpu->kvm, irq); else rc = kvmppc_xics_int_off(vcpu->kvm, irq); @@ -105,7 +105,7 @@ static void kvm_rtas_int_on(struct kvm_vcpu *vcpu, struct rtas_args *args) irq = be32_to_cpu(args->args[0]); - if (xive_enabled()) + if (xics_on_xive()) rc = kvmppc_xive_int_on(vcpu->kvm, irq); else rc = kvmppc_xics_int_on(vcpu->kvm, irq); diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index b90a7d1541800..8885377ec3e0c 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -748,7 +748,7 @@ void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) kvmppc_mpic_disconnect_vcpu(vcpu->arch.mpic, vcpu); break; case KVMPPC_IRQ_XICS: - if (xive_enabled()) + if (xics_on_xive()) kvmppc_xive_cleanup_vcpu(vcpu); else kvmppc_xics_free_icp(vcpu); @@ -1931,7 +1931,7 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, r = -EPERM; dev = kvm_device_from_filp(f.file); if (dev) { - if (xive_enabled()) + if (xics_on_xive()) r = kvmppc_xive_connect_vcpu(dev, vcpu, cap->args[1]); else r = kvmppc_xics_connect_vcpu(dev, vcpu, cap->args[1]); @@ -2189,10 +2189,12 @@ static int pseries_get_cpu_char(struct kvm_ppc_cpu_char *cp) KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV | KVM_PPC_CPU_CHAR_BR_HINT_HONOURED | KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF | - KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS; + KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS | + KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST; cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY | KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR | - KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR; + KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR | + KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE; } return 0; } @@ -2251,12 +2253,16 @@ static int kvmppc_get_cpu_char(struct kvm_ppc_cpu_char *cp) if (have_fw_feat(fw_features, "enabled", "fw-count-cache-disabled")) cp->character |= KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS; + if (have_fw_feat(fw_features, "enabled", + "fw-count-cache-flush-bcctr2,0,0")) + cp->character |= KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST; cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 | KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED | KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 | KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 | KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV | - KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS; + KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS | + KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST; if (have_fw_feat(fw_features, "enabled", "speculation-policy-favor-security")) @@ -2267,9 +2273,13 @@ static int kvmppc_get_cpu_char(struct kvm_ppc_cpu_char *cp) if (!have_fw_feat(fw_features, "disabled", "needs-spec-barrier-for-bound-checks")) cp->behaviour |= KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR; + if (have_fw_feat(fw_features, "enabled", + "needs-count-cache-flush-on-context-switch")) + cp->behaviour |= KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE; cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY | KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR | - KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR; + KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR | + KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE; of_node_put(fw_features); } diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile index d52ec118e09db..3c1bd9fa23cd9 100644 --- a/arch/powerpc/mm/Makefile +++ b/arch/powerpc/mm/Makefile @@ -52,3 +52,6 @@ obj-$(CONFIG_PPC_MEM_KEYS) += pkeys.o # This is necessary for booting with kcov enabled on book3e machines KCOV_INSTRUMENT_tlb_nohash.o := n KCOV_INSTRUMENT_fsl_booke_mmu.o := n + +# Instrumenting the SLB fault path can lead to duplicate SLB entries +KCOV_INSTRUMENT_slb.o := n diff --git a/arch/powerpc/platforms/powernv/opal-call.c b/arch/powerpc/platforms/powernv/opal-call.c index 578757d403ab8..daad8c45c8e72 100644 --- a/arch/powerpc/platforms/powernv/opal-call.c +++ b/arch/powerpc/platforms/powernv/opal-call.c @@ -86,6 +86,7 @@ static s64 __opal_call_trace(s64 a0, s64 a1, s64 a2, s64 a3, s64 a4, s64 a5, s64 a6, s64 a7, unsigned long opcode, unsigned long msr) { + return 0; } #define DO_TRACE false diff --git a/arch/powerpc/platforms/pseries/papr_scm.c b/arch/powerpc/platforms/pseries/papr_scm.c index bba281b1fe1b0..96c53b23e58f9 100644 --- a/arch/powerpc/platforms/pseries/papr_scm.c +++ b/arch/powerpc/platforms/pseries/papr_scm.c @@ -239,6 +239,7 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p) memset(&ndr_desc, 0, sizeof(ndr_desc)); ndr_desc.attr_groups = region_attr_groups; ndr_desc.numa_node = dev_to_node(&p->pdev->dev); + ndr_desc.target_node = ndr_desc.numa_node; ndr_desc.res = &p->res; ndr_desc.of_node = p->dn; ndr_desc.provider_data = p; diff --git a/arch/s390/include/asm/cio.h b/arch/s390/include/asm/cio.h index 2256676520692..1727180e8ca17 100644 --- a/arch/s390/include/asm/cio.h +++ b/arch/s390/include/asm/cio.h @@ -331,5 +331,6 @@ extern void css_schedule_reprobe(void); /* Function from drivers/s390/cio/chsc.c */ int chsc_sstpc(void *page, unsigned int op, u16 ctrl, u64 *clock_delta); int chsc_sstpi(void *page, void *result, size_t size); +int chsc_sgib(u32 origin); #endif diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h index 2f7f27e5493f6..afaf5e3c57fd8 100644 --- a/arch/s390/include/asm/irq.h +++ b/arch/s390/include/asm/irq.h @@ -62,6 +62,7 @@ enum interruption_class { IRQIO_MSI, IRQIO_VIR, IRQIO_VAI, + IRQIO_GAL, NMI_NMI, CPU_RST, NR_ARCH_IRQS diff --git a/arch/s390/include/asm/isc.h b/arch/s390/include/asm/isc.h index 6cb9e2ed05b6c..b2cc1ec78d062 100644 --- a/arch/s390/include/asm/isc.h +++ b/arch/s390/include/asm/isc.h @@ -21,6 +21,7 @@ /* Adapter interrupts. */ #define QDIO_AIRQ_ISC IO_SCH_ISC /* I/O subchannel in qdio mode */ #define PCI_ISC 2 /* PCI I/O subchannels */ +#define GAL_ISC 5 /* GIB alert */ #define AP_ISC 6 /* adjunct processor (crypto) devices */ /* Functions for registration of I/O interruption subclasses */ diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index d5d24889c3bcf..c47e22bba87fa 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -591,7 +591,6 @@ struct kvm_s390_float_interrupt { struct kvm_s390_mchk_info mchk; struct kvm_s390_ext_info srv_signal; int next_rr_cpu; - unsigned long idle_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; struct mutex ais_lock; u8 simm; u8 nimm; @@ -712,6 +711,7 @@ struct s390_io_adapter { struct kvm_s390_cpu_model { /* facility mask supported by kvm & hosting machine */ __u64 fac_mask[S390_ARCH_FAC_LIST_SIZE_U64]; + struct kvm_s390_vm_cpu_subfunc subfuncs; /* facility list requested by guest (in dma page) */ __u64 *fac_list; u64 cpuid; @@ -782,9 +782,21 @@ struct kvm_s390_gisa { u8 reserved03[11]; u32 airq_count; } g1; + struct { + u64 word[4]; + } u64; }; }; +struct kvm_s390_gib { + u32 alert_list_origin; + u32 reserved01; + u8:5; + u8 nisc:3; + u8 reserved03[3]; + u32 reserved04[5]; +}; + /* * sie_page2 has to be allocated as DMA because fac_list, crycb and * gisa need 31bit addresses in the sie control block. @@ -793,7 +805,8 @@ struct sie_page2 { __u64 fac_list[S390_ARCH_FAC_LIST_SIZE_U64]; /* 0x0000 */ struct kvm_s390_crypto_cb crycb; /* 0x0800 */ struct kvm_s390_gisa gisa; /* 0x0900 */ - u8 reserved920[0x1000 - 0x920]; /* 0x0920 */ + struct kvm *kvm; /* 0x0920 */ + u8 reserved928[0x1000 - 0x928]; /* 0x0928 */ }; struct kvm_s390_vsie { @@ -804,6 +817,20 @@ struct kvm_s390_vsie { struct page *pages[KVM_MAX_VCPUS]; }; +struct kvm_s390_gisa_iam { + u8 mask; + spinlock_t ref_lock; + u32 ref_count[MAX_ISC + 1]; +}; + +struct kvm_s390_gisa_interrupt { + struct kvm_s390_gisa *origin; + struct kvm_s390_gisa_iam alert; + struct hrtimer timer; + u64 expires; + DECLARE_BITMAP(kicked_mask, KVM_MAX_VCPUS); +}; + struct kvm_arch{ void *sca; int use_esca; @@ -837,7 +864,8 @@ struct kvm_arch{ atomic64_t cmma_dirty_pages; /* subset of available cpu features enabled by user space */ DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); - struct kvm_s390_gisa *gisa; + DECLARE_BITMAP(idle_mask, KVM_MAX_VCPUS); + struct kvm_s390_gisa_interrupt gisa_int; }; #define KVM_HVA_ERR_BAD (-1UL) @@ -871,6 +899,9 @@ void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm, extern int sie64a(struct kvm_s390_sie_block *, u64 *); extern char sie_exit; +extern int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc); +extern int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc); + static inline void kvm_arch_hardware_disable(void) {} static inline void kvm_arch_check_processor_compat(void *rtn) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} @@ -878,7 +909,7 @@ static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, struct kvm_memory_slot *dont) {} -static inline void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index 0e8d68bac82c2..0cd5a5f96729d 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -88,6 +88,7 @@ static const struct irq_class irqclass_sub_desc[] = { {.irq = IRQIO_MSI, .name = "MSI", .desc = "[I/O] MSI Interrupt" }, {.irq = IRQIO_VIR, .name = "VIR", .desc = "[I/O] Virtual I/O Devices"}, {.irq = IRQIO_VAI, .name = "VAI", .desc = "[I/O] Virtual I/O Devices AI"}, + {.irq = IRQIO_GAL, .name = "GAL", .desc = "[I/O] GIB Alert"}, {.irq = NMI_NMI, .name = "NMI", .desc = "[NMI] Machine Check"}, {.irq = CPU_RST, .name = "RST", .desc = "[CPU] CPU Restart"}, }; diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index fcb55b02990ef..82162867f378d 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -7,6 +7,9 @@ * Author(s): Carsten Otte <cotte@de.ibm.com> */ +#define KMSG_COMPONENT "kvm-s390" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + #include <linux/interrupt.h> #include <linux/kvm_host.h> #include <linux/hrtimer.h> @@ -23,6 +26,7 @@ #include <asm/gmap.h> #include <asm/switch_to.h> #include <asm/nmi.h> +#include <asm/airq.h> #include "kvm-s390.h" #include "gaccess.h" #include "trace-s390.h" @@ -31,6 +35,8 @@ #define PFAULT_DONE 0x0680 #define VIRTIO_PARAM 0x0d00 +static struct kvm_s390_gib *gib; + /* handle external calls via sigp interpretation facility */ static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) { @@ -217,22 +223,100 @@ static inline u8 int_word_to_isc(u32 int_word) */ #define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE) -static inline void kvm_s390_gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) +/** + * gisa_set_iam - change the GISA interruption alert mask + * + * @gisa: gisa to operate on + * @iam: new IAM value to use + * + * Change the IAM atomically with the next alert address and the IPM + * of the GISA if the GISA is not part of the GIB alert list. All three + * fields are located in the first long word of the GISA. + * + * Returns: 0 on success + * -EBUSY in case the gisa is part of the alert list + */ +static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam) +{ + u64 word, _word; + + do { + word = READ_ONCE(gisa->u64.word[0]); + if ((u64)gisa != word >> 32) + return -EBUSY; + _word = (word & ~0xffUL) | iam; + } while (cmpxchg(&gisa->u64.word[0], word, _word) != word); + + return 0; +} + +/** + * gisa_clear_ipm - clear the GISA interruption pending mask + * + * @gisa: gisa to operate on + * + * Clear the IPM atomically with the next alert address and the IAM + * of the GISA unconditionally. All three fields are located in the + * first long word of the GISA. + */ +static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa) +{ + u64 word, _word; + + do { + word = READ_ONCE(gisa->u64.word[0]); + _word = word & ~(0xffUL << 24); + } while (cmpxchg(&gisa->u64.word[0], word, _word) != word); +} + +/** + * gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM + * + * @gi: gisa interrupt struct to work on + * + * Atomically restores the interruption alert mask if none of the + * relevant ISCs are pending and return the IPM. + * + * Returns: the relevant pending ISCs + */ +static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi) +{ + u8 pending_mask, alert_mask; + u64 word, _word; + + do { + word = READ_ONCE(gi->origin->u64.word[0]); + alert_mask = READ_ONCE(gi->alert.mask); + pending_mask = (u8)(word >> 24) & alert_mask; + if (pending_mask) + return pending_mask; + _word = (word & ~0xffUL) | alert_mask; + } while (cmpxchg(&gi->origin->u64.word[0], word, _word) != word); + + return 0; +} + +static inline int gisa_in_alert_list(struct kvm_s390_gisa *gisa) +{ + return READ_ONCE(gisa->next_alert) != (u32)(u64)gisa; +} + +static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) { set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa); } -static inline u8 kvm_s390_gisa_get_ipm(struct kvm_s390_gisa *gisa) +static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa) { return READ_ONCE(gisa->ipm); } -static inline void kvm_s390_gisa_clear_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) +static inline void gisa_clear_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) { clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa); } -static inline int kvm_s390_gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) +static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) { return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa); } @@ -245,8 +329,13 @@ static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu) static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu) { - return pending_irqs_no_gisa(vcpu) | - kvm_s390_gisa_get_ipm(vcpu->kvm->arch.gisa) << IRQ_PEND_IO_ISC_7; + struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int; + unsigned long pending_mask; + + pending_mask = pending_irqs_no_gisa(vcpu); + if (gi->origin) + pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7; + return pending_mask; } static inline int isc_to_irq_type(unsigned long isc) @@ -318,13 +407,13 @@ static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu) static void __set_cpu_idle(struct kvm_vcpu *vcpu) { kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT); - set_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask); + set_bit(vcpu->vcpu_id, vcpu->kvm->arch.idle_mask); } static void __unset_cpu_idle(struct kvm_vcpu *vcpu) { kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT); - clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask); + clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.idle_mask); } static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) @@ -345,7 +434,7 @@ static void set_intercept_indicators_io(struct kvm_vcpu *vcpu) { if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK)) return; - else if (psw_ioint_disabled(vcpu)) + if (psw_ioint_disabled(vcpu)) kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT); else vcpu->arch.sie_block->lctl |= LCTL_CR6; @@ -353,7 +442,7 @@ static void set_intercept_indicators_io(struct kvm_vcpu *vcpu) static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu) { - if (!(pending_irqs(vcpu) & IRQ_PEND_EXT_MASK)) + if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK)) return; if (psw_extint_disabled(vcpu)) kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); @@ -363,7 +452,7 @@ static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu) static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu) { - if (!(pending_irqs(vcpu) & IRQ_PEND_MCHK_MASK)) + if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK)) return; if (psw_mchk_disabled(vcpu)) vcpu->arch.sie_block->ictl |= ICTL_LPSW; @@ -956,6 +1045,7 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu, { struct list_head *isc_list; struct kvm_s390_float_interrupt *fi; + struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int; struct kvm_s390_interrupt_info *inti = NULL; struct kvm_s390_io_info io; u32 isc; @@ -998,8 +1088,7 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu, goto out; } - if (vcpu->kvm->arch.gisa && - kvm_s390_gisa_tac_ipm_gisc(vcpu->kvm->arch.gisa, isc)) { + if (gi->origin && gisa_tac_ipm_gisc(gi->origin, isc)) { /* * in case an adapter interrupt was not delivered * in SIE context KVM will handle the delivery @@ -1089,6 +1178,7 @@ static u64 __calculate_sltime(struct kvm_vcpu *vcpu) int kvm_s390_handle_wait(struct kvm_vcpu *vcpu) { + struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int; u64 sltime; vcpu->stat.exit_wait_state++; @@ -1102,6 +1192,11 @@ int kvm_s390_handle_wait(struct kvm_vcpu *vcpu) return -EOPNOTSUPP; /* disabled wait */ } + if (gi->origin && + (gisa_get_ipm_or_restore_iam(gi) & + vcpu->arch.sie_block->gcr[6] >> 24)) + return 0; + if (!ckc_interrupts_enabled(vcpu) && !cpu_timer_interrupts_enabled(vcpu)) { VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer"); @@ -1533,18 +1628,19 @@ static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm, static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid) { + struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; unsigned long active_mask; int isc; if (schid) goto out; - if (!kvm->arch.gisa) + if (!gi->origin) goto out; - active_mask = (isc_mask & kvm_s390_gisa_get_ipm(kvm->arch.gisa) << 24) << 32; + active_mask = (isc_mask & gisa_get_ipm(gi->origin) << 24) << 32; while (active_mask) { isc = __fls(active_mask) ^ (BITS_PER_LONG - 1); - if (kvm_s390_gisa_tac_ipm_gisc(kvm->arch.gisa, isc)) + if (gisa_tac_ipm_gisc(gi->origin, isc)) return isc; clear_bit_inv(isc, &active_mask); } @@ -1567,6 +1663,7 @@ out: struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, u64 isc_mask, u32 schid) { + struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; struct kvm_s390_interrupt_info *inti, *tmp_inti; int isc; @@ -1584,7 +1681,7 @@ struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, /* both types of interrupts present */ if (int_word_to_isc(inti->io.io_int_word) <= isc) { /* classical IO int with higher priority */ - kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc); + gisa_set_ipm_gisc(gi->origin, isc); goto out; } gisa_out: @@ -1596,7 +1693,7 @@ gisa_out: kvm_s390_reinject_io_int(kvm, inti); inti = tmp_inti; } else - kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc); + gisa_set_ipm_gisc(gi->origin, isc); out: return inti; } @@ -1685,6 +1782,7 @@ static int __inject_float_mchk(struct kvm *kvm, static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { + struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; struct kvm_s390_float_interrupt *fi; struct list_head *list; int isc; @@ -1692,9 +1790,9 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) kvm->stat.inject_io++; isc = int_word_to_isc(inti->io.io_int_word); - if (kvm->arch.gisa && inti->type & KVM_S390_INT_IO_AI_MASK) { + if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) { VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc); - kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc); + gisa_set_ipm_gisc(gi->origin, isc); kfree(inti); return 0; } @@ -1726,7 +1824,6 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) */ static void __floating_irq_kick(struct kvm *kvm, u64 type) { - struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_vcpu *dst_vcpu; int sigcpu, online_vcpus, nr_tries = 0; @@ -1735,11 +1832,11 @@ static void __floating_irq_kick(struct kvm *kvm, u64 type) return; /* find idle VCPUs first, then round robin */ - sigcpu = find_first_bit(fi->idle_mask, online_vcpus); + sigcpu = find_first_bit(kvm->arch.idle_mask, online_vcpus); if (sigcpu == online_vcpus) { do { - sigcpu = fi->next_rr_cpu; - fi->next_rr_cpu = (fi->next_rr_cpu + 1) % online_vcpus; + sigcpu = kvm->arch.float_int.next_rr_cpu++; + kvm->arch.float_int.next_rr_cpu %= online_vcpus; /* avoid endless loops if all vcpus are stopped */ if (nr_tries++ >= online_vcpus) return; @@ -1753,7 +1850,8 @@ static void __floating_irq_kick(struct kvm *kvm, u64 type) kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT); break; case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: - if (!(type & KVM_S390_INT_IO_AI_MASK && kvm->arch.gisa)) + if (!(type & KVM_S390_INT_IO_AI_MASK && + kvm->arch.gisa_int.origin)) kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT); break; default: @@ -2003,6 +2101,7 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm) static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) { + struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; struct kvm_s390_interrupt_info *inti; struct kvm_s390_float_interrupt *fi; struct kvm_s390_irq *buf; @@ -2026,15 +2125,14 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) max_irqs = len / sizeof(struct kvm_s390_irq); - if (kvm->arch.gisa && - kvm_s390_gisa_get_ipm(kvm->arch.gisa)) { + if (gi->origin && gisa_get_ipm(gi->origin)) { for (i = 0; i <= MAX_ISC; i++) { if (n == max_irqs) { /* signal userspace to try again */ ret = -ENOMEM; goto out_nolock; } - if (kvm_s390_gisa_tac_ipm_gisc(kvm->arch.gisa, i)) { + if (gisa_tac_ipm_gisc(gi->origin, i)) { irq = (struct kvm_s390_irq *) &buf[n]; irq->type = KVM_S390_INT_IO(1, 0, 0, 0); irq->u.io.io_int_word = isc_to_int_word(i); @@ -2831,7 +2929,7 @@ static void store_local_irq(struct kvm_s390_local_interrupt *li, int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len) { int scn; - unsigned long sigp_emerg_pending[BITS_TO_LONGS(KVM_MAX_VCPUS)]; + DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS); struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; unsigned long pending_irqs; struct kvm_s390_irq irq; @@ -2884,27 +2982,278 @@ int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len) return n; } -void kvm_s390_gisa_clear(struct kvm *kvm) +static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask) { - if (kvm->arch.gisa) { - memset(kvm->arch.gisa, 0, sizeof(struct kvm_s390_gisa)); - kvm->arch.gisa->next_alert = (u32)(u64)kvm->arch.gisa; - VM_EVENT(kvm, 3, "gisa 0x%pK cleared", kvm->arch.gisa); + int vcpu_id, online_vcpus = atomic_read(&kvm->online_vcpus); + struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; + struct kvm_vcpu *vcpu; + + for_each_set_bit(vcpu_id, kvm->arch.idle_mask, online_vcpus) { + vcpu = kvm_get_vcpu(kvm, vcpu_id); + if (psw_ioint_disabled(vcpu)) + continue; + deliverable_mask &= (u8)(vcpu->arch.sie_block->gcr[6] >> 24); + if (deliverable_mask) { + /* lately kicked but not yet running */ + if (test_and_set_bit(vcpu_id, gi->kicked_mask)) + return; + kvm_s390_vcpu_wakeup(vcpu); + return; + } } } +static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer) +{ + struct kvm_s390_gisa_interrupt *gi = + container_of(timer, struct kvm_s390_gisa_interrupt, timer); + struct kvm *kvm = + container_of(gi->origin, struct sie_page2, gisa)->kvm; + u8 pending_mask; + + pending_mask = gisa_get_ipm_or_restore_iam(gi); + if (pending_mask) { + __airqs_kick_single_vcpu(kvm, pending_mask); + hrtimer_forward_now(timer, ns_to_ktime(gi->expires)); + return HRTIMER_RESTART; + }; + + return HRTIMER_NORESTART; +} + +#define NULL_GISA_ADDR 0x00000000UL +#define NONE_GISA_ADDR 0x00000001UL +#define GISA_ADDR_MASK 0xfffff000UL + +static void process_gib_alert_list(void) +{ + struct kvm_s390_gisa_interrupt *gi; + struct kvm_s390_gisa *gisa; + struct kvm *kvm; + u32 final, origin = 0UL; + + do { + /* + * If the NONE_GISA_ADDR is still stored in the alert list + * origin, we will leave the outer loop. No further GISA has + * been added to the alert list by millicode while processing + * the current alert list. + */ + final = (origin & NONE_GISA_ADDR); + /* + * Cut off the alert list and store the NONE_GISA_ADDR in the + * alert list origin to avoid further GAL interruptions. + * A new alert list can be build up by millicode in parallel + * for guests not in the yet cut-off alert list. When in the + * final loop, store the NULL_GISA_ADDR instead. This will re- + * enable GAL interruptions on the host again. + */ + origin = xchg(&gib->alert_list_origin, + (!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR); + /* + * Loop through the just cut-off alert list and start the + * gisa timers to kick idle vcpus to consume the pending + * interruptions asap. + */ + while (origin & GISA_ADDR_MASK) { + gisa = (struct kvm_s390_gisa *)(u64)origin; + origin = gisa->next_alert; + gisa->next_alert = (u32)(u64)gisa; + kvm = container_of(gisa, struct sie_page2, gisa)->kvm; + gi = &kvm->arch.gisa_int; + if (hrtimer_active(&gi->timer)) + hrtimer_cancel(&gi->timer); + hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL); + } + } while (!final); + +} + +void kvm_s390_gisa_clear(struct kvm *kvm) +{ + struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; + + if (!gi->origin) + return; + gisa_clear_ipm(gi->origin); + VM_EVENT(kvm, 3, "gisa 0x%pK cleared", gi->origin); +} + void kvm_s390_gisa_init(struct kvm *kvm) { - if (css_general_characteristics.aiv) { - kvm->arch.gisa = &kvm->arch.sie_page2->gisa; - VM_EVENT(kvm, 3, "gisa 0x%pK initialized", kvm->arch.gisa); - kvm_s390_gisa_clear(kvm); - } + struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; + + if (!css_general_characteristics.aiv) + return; + gi->origin = &kvm->arch.sie_page2->gisa; + gi->alert.mask = 0; + spin_lock_init(&gi->alert.ref_lock); + gi->expires = 50 * 1000; /* 50 usec */ + hrtimer_init(&gi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + gi->timer.function = gisa_vcpu_kicker; + memset(gi->origin, 0, sizeof(struct kvm_s390_gisa)); + gi->origin->next_alert = (u32)(u64)gi->origin; + VM_EVENT(kvm, 3, "gisa 0x%pK initialized", gi->origin); } void kvm_s390_gisa_destroy(struct kvm *kvm) { - if (!kvm->arch.gisa) + struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; + + if (!gi->origin) + return; + if (gi->alert.mask) + KVM_EVENT(3, "vm 0x%pK has unexpected iam 0x%02x", + kvm, gi->alert.mask); + while (gisa_in_alert_list(gi->origin)) + cpu_relax(); + hrtimer_cancel(&gi->timer); + gi->origin = NULL; +} + +/** + * kvm_s390_gisc_register - register a guest ISC + * + * @kvm: the kernel vm to work with + * @gisc: the guest interruption sub class to register + * + * The function extends the vm specific alert mask to use. + * The effective IAM mask in the GISA is updated as well + * in case the GISA is not part of the GIB alert list. + * It will be updated latest when the IAM gets restored + * by gisa_get_ipm_or_restore_iam(). + * + * Returns: the nonspecific ISC (NISC) the gib alert mechanism + * has registered with the channel subsystem. + * -ENODEV in case the vm uses no GISA + * -ERANGE in case the guest ISC is invalid + */ +int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc) +{ + struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; + + if (!gi->origin) + return -ENODEV; + if (gisc > MAX_ISC) + return -ERANGE; + + spin_lock(&gi->alert.ref_lock); + gi->alert.ref_count[gisc]++; + if (gi->alert.ref_count[gisc] == 1) { + gi->alert.mask |= 0x80 >> gisc; + gisa_set_iam(gi->origin, gi->alert.mask); + } + spin_unlock(&gi->alert.ref_lock); + + return gib->nisc; +} +EXPORT_SYMBOL_GPL(kvm_s390_gisc_register); + +/** + * kvm_s390_gisc_unregister - unregister a guest ISC + * + * @kvm: the kernel vm to work with + * @gisc: the guest interruption sub class to register + * + * The function reduces the vm specific alert mask to use. + * The effective IAM mask in the GISA is updated as well + * in case the GISA is not part of the GIB alert list. + * It will be updated latest when the IAM gets restored + * by gisa_get_ipm_or_restore_iam(). + * + * Returns: the nonspecific ISC (NISC) the gib alert mechanism + * has registered with the channel subsystem. + * -ENODEV in case the vm uses no GISA + * -ERANGE in case the guest ISC is invalid + * -EINVAL in case the guest ISC is not registered + */ +int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc) +{ + struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; + int rc = 0; + + if (!gi->origin) + return -ENODEV; + if (gisc > MAX_ISC) + return -ERANGE; + + spin_lock(&gi->alert.ref_lock); + if (gi->alert.ref_count[gisc] == 0) { + rc = -EINVAL; + goto out; + } + gi->alert.ref_count[gisc]--; + if (gi->alert.ref_count[gisc] == 0) { + gi->alert.mask &= ~(0x80 >> gisc); + gisa_set_iam(gi->origin, gi->alert.mask); + } +out: + spin_unlock(&gi->alert.ref_lock); + + return rc; +} +EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister); + +static void gib_alert_irq_handler(struct airq_struct *airq) +{ + inc_irq_stat(IRQIO_GAL); + process_gib_alert_list(); +} + +static struct airq_struct gib_alert_irq = { + .handler = gib_alert_irq_handler, + .lsi_ptr = &gib_alert_irq.lsi_mask, +}; + +void kvm_s390_gib_destroy(void) +{ + if (!gib) return; - kvm->arch.gisa = NULL; + chsc_sgib(0); + unregister_adapter_interrupt(&gib_alert_irq); + free_page((unsigned long)gib); + gib = NULL; +} + +int kvm_s390_gib_init(u8 nisc) +{ + int rc = 0; + + if (!css_general_characteristics.aiv) { + KVM_EVENT(3, "%s", "gib not initialized, no AIV facility"); + goto out; + } + + gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + if (!gib) { + rc = -ENOMEM; + goto out; + } + + gib_alert_irq.isc = nisc; + if (register_adapter_interrupt(&gib_alert_irq)) { + pr_err("Registering the GIB alert interruption handler failed\n"); + rc = -EIO; + goto out_free_gib; + } + + gib->nisc = nisc; + if (chsc_sgib((u32)(u64)gib)) { + pr_err("Associating the GIB with the AIV facility failed\n"); + free_page((unsigned long)gib); + gib = NULL; + rc = -EIO; + goto out_unreg_gal; + } + + KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc); + goto out; + +out_unreg_gal: + unregister_adapter_interrupt(&gib_alert_irq); +out_free_gib: + free_page((unsigned long)gib); + gib = NULL; +out: + return rc; } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 7f4bc58a53b97..4638303ba6a85 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -432,11 +432,18 @@ int kvm_arch_init(void *opaque) /* Register floating interrupt controller interface. */ rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC); if (rc) { - pr_err("Failed to register FLIC rc=%d\n", rc); + pr_err("A FLIC registration call failed with rc=%d\n", rc); goto out_debug_unreg; } + + rc = kvm_s390_gib_init(GAL_ISC); + if (rc) + goto out_gib_destroy; + return 0; +out_gib_destroy: + kvm_s390_gib_destroy(); out_debug_unreg: debug_unregister(kvm_s390_dbf); return rc; @@ -444,6 +451,7 @@ out_debug_unreg: void kvm_arch_exit(void) { + kvm_s390_gib_destroy(); debug_unregister(kvm_s390_dbf); } @@ -1258,11 +1266,65 @@ static int kvm_s390_set_processor_feat(struct kvm *kvm, static int kvm_s390_set_processor_subfunc(struct kvm *kvm, struct kvm_device_attr *attr) { - /* - * Once supported by kernel + hw, we have to store the subfunctions - * in kvm->arch and remember that user space configured them. - */ - return -ENXIO; + mutex_lock(&kvm->lock); + if (kvm->created_vcpus) { + mutex_unlock(&kvm->lock); + return -EBUSY; + } + + if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr, + sizeof(struct kvm_s390_vm_cpu_subfunc))) { + mutex_unlock(&kvm->lock); + return -EFAULT; + } + mutex_unlock(&kvm->lock); + + VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1], + ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2], + ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]); + VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]); + VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]); + VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]); + VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.km)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]); + VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]); + VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]); + VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]); + VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]); + VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]); + VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]); + VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]); + VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]); + VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]); + + return 0; } static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr) @@ -1381,12 +1443,56 @@ static int kvm_s390_get_machine_feat(struct kvm *kvm, static int kvm_s390_get_processor_subfunc(struct kvm *kvm, struct kvm_device_attr *attr) { - /* - * Once we can actually configure subfunctions (kernel + hw support), - * we have to check if they were already set by user space, if so copy - * them from kvm->arch. - */ - return -ENXIO; + if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs, + sizeof(struct kvm_s390_vm_cpu_subfunc))) + return -EFAULT; + + VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1], + ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2], + ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]); + VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]); + VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]); + VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]); + VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.km)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]); + VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]); + VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]); + VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]); + VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]); + VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]); + VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]); + VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]); + VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]); + VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0], + ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]); + + return 0; } static int kvm_s390_get_machine_subfunc(struct kvm *kvm, @@ -1395,8 +1501,55 @@ static int kvm_s390_get_machine_subfunc(struct kvm *kvm, if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc, sizeof(struct kvm_s390_vm_cpu_subfunc))) return -EFAULT; + + VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.plo)[0], + ((unsigned long *) &kvm_s390_available_subfunc.plo)[1], + ((unsigned long *) &kvm_s390_available_subfunc.plo)[2], + ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]); + VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0], + ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]); + VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0], + ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]); + VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0], + ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]); + VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.km)[0], + ((unsigned long *) &kvm_s390_available_subfunc.km)[1]); + VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0], + ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]); + VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0], + ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]); + VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0], + ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]); + VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0], + ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]); + VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0], + ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]); + VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0], + ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]); + VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0], + ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]); + VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0], + ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]); + VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx", + ((unsigned long *) &kvm_s390_available_subfunc.kma)[0], + ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]); + return 0; } + static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr) { int ret = -ENXIO; @@ -1514,10 +1667,9 @@ static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) case KVM_S390_VM_CPU_PROCESSOR_FEAT: case KVM_S390_VM_CPU_MACHINE_FEAT: case KVM_S390_VM_CPU_MACHINE_SUBFUNC: + case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC: ret = 0; break; - /* configuring subfunctions is not supported yet */ - case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC: default: ret = -ENXIO; break; @@ -2209,6 +2361,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) if (!kvm->arch.sie_page2) goto out_err; + kvm->arch.sie_page2->kvm = kvm; kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list; for (i = 0; i < kvm_s390_fac_size(); i++) { @@ -2218,6 +2371,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] & kvm_s390_fac_base[i]; } + kvm->arch.model.subfuncs = kvm_s390_available_subfunc; /* we are always in czam mode - even on pre z14 machines */ set_kvm_facility(kvm->arch.model.fac_mask, 138); @@ -2812,7 +2966,7 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, vcpu->arch.sie_block->icpua = id; spin_lock_init(&vcpu->arch.local_int.lock); - vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa; + vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa_int.origin; if (vcpu->arch.sie_block->gd && sclp.has_gisaf) vcpu->arch.sie_block->gd |= GISA_FORMAT1; seqcount_init(&vcpu->arch.cputm_seqcount); @@ -3458,6 +3612,8 @@ static int vcpu_pre_run(struct kvm_vcpu *vcpu) kvm_s390_patch_guest_per_regs(vcpu); } + clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.gisa_int.kicked_mask); + vcpu->arch.sie_block->icptcode = 0; cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags); VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags); @@ -4293,12 +4449,12 @@ static int __init kvm_s390_init(void) int i; if (!sclp.has_sief2) { - pr_info("SIE not available\n"); + pr_info("SIE is not available\n"); return -ENODEV; } if (nested && hpage) { - pr_info("nested (vSIE) and hpage (huge page backing) can currently not be activated concurrently"); + pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n"); return -EINVAL; } diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index 1f6e36cdce0da..6d9448dbd052b 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -67,7 +67,7 @@ static inline int is_vcpu_stopped(struct kvm_vcpu *vcpu) static inline int is_vcpu_idle(struct kvm_vcpu *vcpu) { - return test_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask); + return test_bit(vcpu->vcpu_id, vcpu->kvm->arch.idle_mask); } static inline int kvm_is_ucontrol(struct kvm *kvm) @@ -381,6 +381,8 @@ int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, void kvm_s390_gisa_init(struct kvm *kvm); void kvm_s390_gisa_clear(struct kvm *kvm); void kvm_s390_gisa_destroy(struct kvm *kvm); +int kvm_s390_gib_init(u8 nisc); +void kvm_s390_gib_destroy(void); /* implemented in guestdbg.c */ void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu); diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h index 88fe4f978acab..9265a9eece15f 100644 --- a/arch/sparc/include/uapi/asm/socket.h +++ b/arch/sparc/include/uapi/asm/socket.h @@ -2,8 +2,8 @@ #ifndef _ASM_SOCKET_H #define _ASM_SOCKET_H +#include <linux/posix_types.h> #include <asm/sockios.h> -#include <asm/bitsperlong.h> /* For setsockopt(2) */ #define SOL_SOCKET 0xffff diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c index a4a41421c5e2a..aca09be2373e7 100644 --- a/arch/um/drivers/ubd_kern.c +++ b/arch/um/drivers/ubd_kern.c @@ -938,7 +938,7 @@ static int ubd_add(int n, char **error_out) ubd_dev->queue = blk_mq_init_queue(&ubd_dev->tag_set); if (IS_ERR(ubd_dev->queue)) { err = PTR_ERR(ubd_dev->queue); - goto out_cleanup; + goto out_cleanup_tags; } ubd_dev->queue->queuedata = ubd_dev; @@ -968,8 +968,8 @@ out: out_cleanup_tags: blk_mq_free_tag_set(&ubd_dev->tag_set); -out_cleanup: - blk_cleanup_queue(ubd_dev->queue); + if (!(IS_ERR(ubd_dev->queue))) + blk_cleanup_queue(ubd_dev->queue); goto out; } diff --git a/arch/um/drivers/vector_user.c b/arch/um/drivers/vector_user.c index d2c17dd746204..b3f7b3ca896dd 100644 --- a/arch/um/drivers/vector_user.c +++ b/arch/um/drivers/vector_user.c @@ -16,14 +16,12 @@ #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> -#include <sys/types.h> #include <sys/socket.h> #include <net/ethernet.h> #include <netinet/ip.h> #include <netinet/ether.h> #include <linux/if_ether.h> #include <linux/if_packet.h> -#include <sys/socket.h> #include <sys/wait.h> #include <sys/uio.h> #include <linux/virtio_net.h> @@ -31,7 +29,6 @@ #include <stdlib.h> #include <os.h> #include <um_malloc.h> -#include <sys/uio.h> #include "vector_user.h" #define ID_GRE 0 diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 8da78595d69db..1f9607ed087c0 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -429,6 +429,7 @@ 421 i386 rt_sigtimedwait_time64 sys_rt_sigtimedwait __ia32_compat_sys_rt_sigtimedwait_time64 422 i386 futex_time64 sys_futex __ia32_sys_futex 423 i386 sched_rr_get_interval_time64 sys_sched_rr_get_interval __ia32_sys_sched_rr_get_interval +424 i386 pidfd_send_signal sys_pidfd_send_signal __ia32_sys_pidfd_send_signal 425 i386 io_uring_setup sys_io_uring_setup __ia32_sys_io_uring_setup 426 i386 io_uring_enter sys_io_uring_enter __ia32_sys_io_uring_enter 427 i386 io_uring_register sys_io_uring_register __ia32_sys_io_uring_register diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index c768447f97ec6..92ee0b4378d4c 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -345,6 +345,7 @@ 334 common rseq __x64_sys_rseq # don't use numbers 387 through 423, add new calls after the last # 'common' entry +424 common pidfd_send_signal __x64_sys_pidfd_send_signal 425 common io_uring_setup __x64_sys_io_uring_setup 426 common io_uring_enter __x64_sys_io_uring_enter 427 common io_uring_register __x64_sys_io_uring_register diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 35102ecdfc8d5..8baa441d8000f 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3410,7 +3410,7 @@ tfa_get_event_constraints(struct cpu_hw_events *cpuc, int idx, /* * Without TFA we must not use PMC3. */ - if (!allow_tsx_force_abort && test_bit(3, c->idxmsk)) { + if (!allow_tsx_force_abort && test_bit(3, c->idxmsk) && idx >= 0) { c = dyn_constraint(cpuc, c, idx); c->idxmsk64 &= ~(1ULL << 3); c->weight--; @@ -4179,7 +4179,7 @@ static struct attribute *intel_pmu_caps_attrs[] = { NULL }; -DEVICE_BOOL_ATTR(allow_tsx_force_abort, 0644, allow_tsx_force_abort); +static DEVICE_BOOL_ATTR(allow_tsx_force_abort, 0644, allow_tsx_force_abort); static struct attribute *intel_pmu_attrs[] = { &dev_attr_freeze_on_smi.attr, diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index b04ae6c8775e0..a75955741c504 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1033,12 +1033,12 @@ static inline int intel_pmu_init(void) return 0; } -static inline int intel_cpuc_prepare(struct cpu_hw_event *cpuc, int cpu) +static inline int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu) { return 0; } -static inline void intel_cpuc_finish(struct cpu_hw_event *cpuc) +static inline void intel_cpuc_finish(struct cpu_hw_events *cpuc) { } diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index ad7b210aa3f62..d153d570bb047 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -36,13 +36,7 @@ * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1). */ -#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1) -/* Technically wrong, but this avoids compilation errors on some gcc - versions. */ -#define BITOP_ADDR(x) "=m" (*(volatile long *) (x)) -#else #define BITOP_ADDR(x) "+m" (*(volatile long *) (x)) -#endif #define ADDR BITOP_ADDR(addr) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 180373360e342..a5db4475e72db 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -35,6 +35,7 @@ #include <asm/msr-index.h> #include <asm/asm.h> #include <asm/kvm_page_track.h> +#include <asm/kvm_vcpu_regs.h> #include <asm/hyperv-tlfs.h> #define KVM_MAX_VCPUS 288 @@ -137,23 +138,23 @@ static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level) #define ASYNC_PF_PER_VCPU 64 enum kvm_reg { - VCPU_REGS_RAX = 0, - VCPU_REGS_RCX = 1, - VCPU_REGS_RDX = 2, - VCPU_REGS_RBX = 3, - VCPU_REGS_RSP = 4, - VCPU_REGS_RBP = 5, - VCPU_REGS_RSI = 6, - VCPU_REGS_RDI = 7, + VCPU_REGS_RAX = __VCPU_REGS_RAX, + VCPU_REGS_RCX = __VCPU_REGS_RCX, + VCPU_REGS_RDX = __VCPU_REGS_RDX, + VCPU_REGS_RBX = __VCPU_REGS_RBX, + VCPU_REGS_RSP = __VCPU_REGS_RSP, + VCPU_REGS_RBP = __VCPU_REGS_RBP, + VCPU_REGS_RSI = __VCPU_REGS_RSI, + VCPU_REGS_RDI = __VCPU_REGS_RDI, #ifdef CONFIG_X86_64 - VCPU_REGS_R8 = 8, - VCPU_REGS_R9 = 9, - VCPU_REGS_R10 = 10, - VCPU_REGS_R11 = 11, - VCPU_REGS_R12 = 12, - VCPU_REGS_R13 = 13, - VCPU_REGS_R14 = 14, - VCPU_REGS_R15 = 15, + VCPU_REGS_R8 = __VCPU_REGS_R8, + VCPU_REGS_R9 = __VCPU_REGS_R9, + VCPU_REGS_R10 = __VCPU_REGS_R10, + VCPU_REGS_R11 = __VCPU_REGS_R11, + VCPU_REGS_R12 = __VCPU_REGS_R12, + VCPU_REGS_R13 = __VCPU_REGS_R13, + VCPU_REGS_R14 = __VCPU_REGS_R14, + VCPU_REGS_R15 = __VCPU_REGS_R15, #endif VCPU_REGS_RIP, NR_VCPU_REGS @@ -319,6 +320,7 @@ struct kvm_mmu_page { struct list_head link; struct hlist_node hash_link; bool unsync; + bool mmio_cached; /* * The following two entries are used to key the shadow page in the @@ -333,10 +335,6 @@ struct kvm_mmu_page { int root_count; /* Currently serving as active root */ unsigned int unsync_children; struct kvm_rmap_head parent_ptes; /* rmap pointers to parent sptes */ - - /* The page is obsolete if mmu_valid_gen != kvm->arch.mmu_valid_gen. */ - unsigned long mmu_valid_gen; - DECLARE_BITMAP(unsync_child_bitmap, 512); #ifdef CONFIG_X86_32 @@ -848,13 +846,11 @@ struct kvm_arch { unsigned int n_requested_mmu_pages; unsigned int n_max_mmu_pages; unsigned int indirect_shadow_pages; - unsigned long mmu_valid_gen; struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; /* * Hash table of struct kvm_mmu_page. */ struct list_head active_mmu_pages; - struct list_head zapped_obsolete_pages; struct kvm_page_track_notifier_node mmu_sp_tracker; struct kvm_page_track_notifier_head track_notifier_head; @@ -1255,7 +1251,7 @@ void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn_offset, unsigned long mask); void kvm_mmu_zap_all(struct kvm *kvm); -void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots); +void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen); unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm); void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); diff --git a/arch/x86/include/asm/kvm_vcpu_regs.h b/arch/x86/include/asm/kvm_vcpu_regs.h new file mode 100644 index 0000000000000..1af2cb59233b5 --- /dev/null +++ b/arch/x86/include/asm/kvm_vcpu_regs.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_KVM_VCPU_REGS_H +#define _ASM_X86_KVM_VCPU_REGS_H + +#define __VCPU_REGS_RAX 0 +#define __VCPU_REGS_RCX 1 +#define __VCPU_REGS_RDX 2 +#define __VCPU_REGS_RBX 3 +#define __VCPU_REGS_RSP 4 +#define __VCPU_REGS_RBP 5 +#define __VCPU_REGS_RSI 6 +#define __VCPU_REGS_RDI 7 + +#ifdef CONFIG_X86_64 +#define __VCPU_REGS_R8 8 +#define __VCPU_REGS_R9 9 +#define __VCPU_REGS_R10 10 +#define __VCPU_REGS_R11 11 +#define __VCPU_REGS_R12 12 +#define __VCPU_REGS_R13 13 +#define __VCPU_REGS_R14 14 +#define __VCPU_REGS_R15 15 +#endif + +#endif /* _ASM_X86_KVM_VCPU_REGS_H */ diff --git a/arch/x86/include/asm/string_32.h b/arch/x86/include/asm/string_32.h index 55d392c6bd29b..f74362b056199 100644 --- a/arch/x86/include/asm/string_32.h +++ b/arch/x86/include/asm/string_32.h @@ -179,14 +179,7 @@ static inline void *__memcpy3d(void *to, const void *from, size_t len) * No 3D Now! */ -#if (__GNUC__ >= 4) #define memcpy(t, f, n) __builtin_memcpy(t, f, n) -#else -#define memcpy(t, f, n) \ - (__builtin_constant_p((n)) \ - ? __constant_memcpy((t), (f), (n)) \ - : __memcpy((t), (f), (n))) -#endif #endif #endif /* !CONFIG_FORTIFY_SOURCE */ @@ -216,29 +209,6 @@ static inline void *__memset_generic(void *s, char c, size_t count) /* we might want to write optimized versions of these later */ #define __constant_count_memset(s, c, count) __memset_generic((s), (c), (count)) -/* - * memset(x, 0, y) is a reasonably common thing to do, so we want to fill - * things 32 bits at a time even when we don't know the size of the - * area at compile-time.. - */ -static __always_inline -void *__constant_c_memset(void *s, unsigned long c, size_t count) -{ - int d0, d1; - asm volatile("rep ; stosl\n\t" - "testb $2,%b3\n\t" - "je 1f\n\t" - "stosw\n" - "1:\ttestb $1,%b3\n\t" - "je 2f\n\t" - "stosb\n" - "2:" - : "=&c" (d0), "=&D" (d1) - : "a" (c), "q" (count), "0" (count/4), "1" ((long)s) - : "memory"); - return s; -} - /* Added by Gertjan van Wingerde to make minix and sysv module work */ #define __HAVE_ARCH_STRNLEN extern size_t strnlen(const char *s, size_t count); @@ -247,72 +217,6 @@ extern size_t strnlen(const char *s, size_t count); #define __HAVE_ARCH_STRSTR extern char *strstr(const char *cs, const char *ct); -/* - * This looks horribly ugly, but the compiler can optimize it totally, - * as we by now know that both pattern and count is constant.. - */ -static __always_inline -void *__constant_c_and_count_memset(void *s, unsigned long pattern, - size_t count) -{ - switch (count) { - case 0: - return s; - case 1: - *(unsigned char *)s = pattern & 0xff; - return s; - case 2: - *(unsigned short *)s = pattern & 0xffff; - return s; - case 3: - *(unsigned short *)s = pattern & 0xffff; - *((unsigned char *)s + 2) = pattern & 0xff; - return s; - case 4: - *(unsigned long *)s = pattern; - return s; - } - -#define COMMON(x) \ - asm volatile("rep ; stosl" \ - x \ - : "=&c" (d0), "=&D" (d1) \ - : "a" (eax), "0" (count/4), "1" ((long)s) \ - : "memory") - - { - int d0, d1; -#if __GNUC__ == 4 && __GNUC_MINOR__ == 0 - /* Workaround for broken gcc 4.0 */ - register unsigned long eax asm("%eax") = pattern; -#else - unsigned long eax = pattern; -#endif - - switch (count % 4) { - case 0: - COMMON(""); - return s; - case 1: - COMMON("\n\tstosb"); - return s; - case 2: - COMMON("\n\tstosw"); - return s; - default: - COMMON("\n\tstosw\n\tstosb"); - return s; - } - } - -#undef COMMON -} - -#define __constant_c_x_memset(s, c, count) \ - (__builtin_constant_p(count) \ - ? __constant_c_and_count_memset((s), (c), (count)) \ - : __constant_c_memset((s), (c), (count))) - #define __memset(s, c, count) \ (__builtin_constant_p(count) \ ? __constant_count_memset((s), (c), (count)) \ @@ -321,15 +225,7 @@ void *__constant_c_and_count_memset(void *s, unsigned long pattern, #define __HAVE_ARCH_MEMSET extern void *memset(void *, int, size_t); #ifndef CONFIG_FORTIFY_SOURCE -#if (__GNUC__ >= 4) #define memset(s, c, count) __builtin_memset(s, c, count) -#else -#define memset(s, c, count) \ - (__builtin_constant_p(c) \ - ? __constant_c_x_memset((s), (0x01010101UL * (unsigned char)(c)), \ - (count)) \ - : __memset((s), (c), (count))) -#endif #endif /* !CONFIG_FORTIFY_SOURCE */ #define __HAVE_ARCH_MEMSET16 diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h index 4e4194e21a097..75314c3dbe471 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -14,21 +14,6 @@ extern void *memcpy(void *to, const void *from, size_t len); extern void *__memcpy(void *to, const void *from, size_t len); -#ifndef CONFIG_FORTIFY_SOURCE -#if (__GNUC__ == 4 && __GNUC_MINOR__ < 3) || __GNUC__ < 4 -#define memcpy(dst, src, len) \ -({ \ - size_t __len = (len); \ - void *__ret; \ - if (__builtin_constant_p(len) && __len >= 64) \ - __ret = __memcpy((dst), (src), __len); \ - else \ - __ret = __builtin_memcpy((dst), (src), __len); \ - __ret; \ -}) -#endif -#endif /* !CONFIG_FORTIFY_SOURCE */ - #define __HAVE_ARCH_MEMSET void *memset(void *s, int c, size_t n); void *__memset(void *s, int c, size_t n); diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index e811d4d1c8247..904494b924c13 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -104,12 +104,8 @@ static u64 kvm_sched_clock_read(void) static inline void kvm_sched_clock_init(bool stable) { - if (!stable) { - pv_ops.time.sched_clock = kvm_clock_read; + if (!stable) clear_sched_clock_stable(); - return; - } - kvm_sched_clock_offset = kvm_clock_read(); pv_ops.time.sched_clock = kvm_sched_clock_read; @@ -355,6 +351,20 @@ void __init kvmclock_init(void) machine_ops.crash_shutdown = kvm_crash_shutdown; #endif kvm_get_preset_lpj(); + + /* + * X86_FEATURE_NONSTOP_TSC is TSC runs at constant rate + * with P/T states and does not stop in deep C-states. + * + * Invariant TSC exposed by host means kvmclock is not necessary: + * can use TSC as clocksource. + * + */ + if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC) && + boot_cpu_has(X86_FEATURE_NONSTOP_TSC) && + !check_tsc_unstable()) + kvm_clock.rating = 299; + clocksource_register_hz(&kvm_clock, NSEC_PER_SEC); pv_info.name = "KVM"; } diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index c07958b59f505..fd3951638ae45 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -405,7 +405,7 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, F(AVX512VBMI) | F(LA57) | F(PKU) | 0 /*OSPKE*/ | F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) | F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG) | - F(CLDEMOTE); + F(CLDEMOTE) | F(MOVDIRI) | F(MOVDIR64B); /* cpuid 7.0.edx*/ const u32 kvm_cpuid_7_0_edx_x86_features = diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 89d20ed1d2e8b..27c43525a05f1 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1729,7 +1729,7 @@ static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd) mutex_lock(&hv->hv_lock); ret = idr_alloc(&hv->conn_to_evt, eventfd, conn_id, conn_id + 1, - GFP_KERNEL); + GFP_KERNEL_ACCOUNT); mutex_unlock(&hv->hv_lock); if (ret >= 0) diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c index af192895b1fc6..4a6dc54cc12be 100644 --- a/arch/x86/kvm/i8254.c +++ b/arch/x86/kvm/i8254.c @@ -653,7 +653,7 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags) pid_t pid_nr; int ret; - pit = kzalloc(sizeof(struct kvm_pit), GFP_KERNEL); + pit = kzalloc(sizeof(struct kvm_pit), GFP_KERNEL_ACCOUNT); if (!pit) return NULL; diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c index bdcd4139eca92..8b38bb4868a65 100644 --- a/arch/x86/kvm/i8259.c +++ b/arch/x86/kvm/i8259.c @@ -583,7 +583,7 @@ int kvm_pic_init(struct kvm *kvm) struct kvm_pic *s; int ret; - s = kzalloc(sizeof(struct kvm_pic), GFP_KERNEL); + s = kzalloc(sizeof(struct kvm_pic), GFP_KERNEL_ACCOUNT); if (!s) return -ENOMEM; spin_lock_init(&s->lock); diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c index 4e822ad363f37..1add1bc881e22 100644 --- a/arch/x86/kvm/ioapic.c +++ b/arch/x86/kvm/ioapic.c @@ -622,7 +622,7 @@ int kvm_ioapic_init(struct kvm *kvm) struct kvm_ioapic *ioapic; int ret; - ioapic = kzalloc(sizeof(struct kvm_ioapic), GFP_KERNEL); + ioapic = kzalloc(sizeof(struct kvm_ioapic), GFP_KERNEL_ACCOUNT); if (!ioapic) return -ENOMEM; spin_lock_init(&ioapic->lock); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 4b6c2da7265c8..991fdf7fc17fb 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -181,7 +181,8 @@ static void recalculate_apic_map(struct kvm *kvm) max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); new = kvzalloc(sizeof(struct kvm_apic_map) + - sizeof(struct kvm_lapic *) * ((u64)max_id + 1), GFP_KERNEL); + sizeof(struct kvm_lapic *) * ((u64)max_id + 1), + GFP_KERNEL_ACCOUNT); if (!new) goto out; @@ -2259,13 +2260,13 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu) ASSERT(vcpu != NULL); apic_debug("apic_init %d\n", vcpu->vcpu_id); - apic = kzalloc(sizeof(*apic), GFP_KERNEL); + apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT); if (!apic) goto nomem; vcpu->arch.apic = apic; - apic->regs = (void *)get_zeroed_page(GFP_KERNEL); + apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); if (!apic->regs) { printk(KERN_ERR "malloc apic regs error for vcpu %x\n", vcpu->vcpu_id); diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index f2d1d230d5b84..7837ab001d806 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -109,9 +109,11 @@ module_param(dbg, bool, 0644); (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1)) -#define PT64_BASE_ADDR_MASK __sme_clr((((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))) -#define PT64_DIR_BASE_ADDR_MASK \ - (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1)) +#ifdef CONFIG_DYNAMIC_PHYSICAL_MASK +#define PT64_BASE_ADDR_MASK (physical_mask & ~(u64)(PAGE_SIZE-1)) +#else +#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1)) +#endif #define PT64_LVL_ADDR_MASK(level) \ (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \ * PT64_LEVEL_BITS))) - 1)) @@ -330,53 +332,56 @@ static inline bool is_access_track_spte(u64 spte) } /* - * the low bit of the generation number is always presumed to be zero. - * This disables mmio caching during memslot updates. The concept is - * similar to a seqcount but instead of retrying the access we just punt - * and ignore the cache. + * Due to limited space in PTEs, the MMIO generation is a 19 bit subset of + * the memslots generation and is derived as follows: * - * spte bits 3-11 are used as bits 1-9 of the generation number, - * the bits 52-61 are used as bits 10-19 of the generation number. + * Bits 0-8 of the MMIO generation are propagated to spte bits 3-11 + * Bits 9-18 of the MMIO generation are propagated to spte bits 52-61 + * + * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in + * the MMIO generation number, as doing so would require stealing a bit from + * the "real" generation number and thus effectively halve the maximum number + * of MMIO generations that can be handled before encountering a wrap (which + * requires a full MMU zap). The flag is instead explicitly queried when + * checking for MMIO spte cache hits. */ -#define MMIO_SPTE_GEN_LOW_SHIFT 2 -#define MMIO_SPTE_GEN_HIGH_SHIFT 52 +#define MMIO_SPTE_GEN_MASK GENMASK_ULL(18, 0) -#define MMIO_GEN_SHIFT 20 -#define MMIO_GEN_LOW_SHIFT 10 -#define MMIO_GEN_LOW_MASK ((1 << MMIO_GEN_LOW_SHIFT) - 2) -#define MMIO_GEN_MASK ((1 << MMIO_GEN_SHIFT) - 1) +#define MMIO_SPTE_GEN_LOW_START 3 +#define MMIO_SPTE_GEN_LOW_END 11 +#define MMIO_SPTE_GEN_LOW_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \ + MMIO_SPTE_GEN_LOW_START) -static u64 generation_mmio_spte_mask(unsigned int gen) +#define MMIO_SPTE_GEN_HIGH_START 52 +#define MMIO_SPTE_GEN_HIGH_END 61 +#define MMIO_SPTE_GEN_HIGH_MASK GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \ + MMIO_SPTE_GEN_HIGH_START) +static u64 generation_mmio_spte_mask(u64 gen) { u64 mask; - WARN_ON(gen & ~MMIO_GEN_MASK); + WARN_ON(gen & ~MMIO_SPTE_GEN_MASK); - mask = (gen & MMIO_GEN_LOW_MASK) << MMIO_SPTE_GEN_LOW_SHIFT; - mask |= ((u64)gen >> MMIO_GEN_LOW_SHIFT) << MMIO_SPTE_GEN_HIGH_SHIFT; + mask = (gen << MMIO_SPTE_GEN_LOW_START) & MMIO_SPTE_GEN_LOW_MASK; + mask |= (gen << MMIO_SPTE_GEN_HIGH_START) & MMIO_SPTE_GEN_HIGH_MASK; return mask; } -static unsigned int get_mmio_spte_generation(u64 spte) +static u64 get_mmio_spte_generation(u64 spte) { - unsigned int gen; + u64 gen; spte &= ~shadow_mmio_mask; - gen = (spte >> MMIO_SPTE_GEN_LOW_SHIFT) & MMIO_GEN_LOW_MASK; - gen |= (spte >> MMIO_SPTE_GEN_HIGH_SHIFT) << MMIO_GEN_LOW_SHIFT; + gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_START; + gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_START; return gen; } -static unsigned int kvm_current_mmio_generation(struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_memslots(vcpu)->generation & MMIO_GEN_MASK; -} - static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn, unsigned access) { - unsigned int gen = kvm_current_mmio_generation(vcpu); + u64 gen = kvm_vcpu_memslots(vcpu)->generation & MMIO_SPTE_GEN_MASK; u64 mask = generation_mmio_spte_mask(gen); u64 gpa = gfn << PAGE_SHIFT; @@ -386,6 +391,8 @@ static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn, mask |= (gpa & shadow_nonpresent_or_rsvd_mask) << shadow_nonpresent_or_rsvd_mask_len; + page_header(__pa(sptep))->mmio_cached = true; + trace_mark_mmio_spte(sptep, gfn, access, gen); mmu_spte_set(sptep, mask); } @@ -407,7 +414,7 @@ static gfn_t get_mmio_spte_gfn(u64 spte) static unsigned get_mmio_spte_access(u64 spte) { - u64 mask = generation_mmio_spte_mask(MMIO_GEN_MASK) | shadow_mmio_mask; + u64 mask = generation_mmio_spte_mask(MMIO_SPTE_GEN_MASK) | shadow_mmio_mask; return (spte & ~mask) & ~PAGE_MASK; } @@ -424,9 +431,13 @@ static bool set_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte) { - unsigned int kvm_gen, spte_gen; + u64 kvm_gen, spte_gen, gen; - kvm_gen = kvm_current_mmio_generation(vcpu); + gen = kvm_vcpu_memslots(vcpu)->generation; + if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS)) + return false; + + kvm_gen = gen & MMIO_SPTE_GEN_MASK; spte_gen = get_mmio_spte_generation(spte); trace_check_mmio_spte(spte, kvm_gen, spte_gen); @@ -959,7 +970,7 @@ static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, if (cache->nobjs >= min) return 0; while (cache->nobjs < ARRAY_SIZE(cache->objects)) { - obj = kmem_cache_zalloc(base_cache, GFP_KERNEL); + obj = kmem_cache_zalloc(base_cache, GFP_KERNEL_ACCOUNT); if (!obj) return cache->nobjs >= min ? 0 : -ENOMEM; cache->objects[cache->nobjs++] = obj; @@ -2049,12 +2060,6 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct if (!direct) sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache); set_page_private(virt_to_page(sp->spt), (unsigned long)sp); - - /* - * The active_mmu_pages list is the FIFO list, do not move the - * page until it is zapped. kvm_zap_obsolete_pages depends on - * this feature. See the comments in kvm_zap_obsolete_pages(). - */ list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages); kvm_mod_used_mmu_pages(vcpu->kvm, +1); return sp; @@ -2195,23 +2200,15 @@ static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp) --kvm->stat.mmu_unsync; } -static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, - struct list_head *invalid_list); +static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, + struct list_head *invalid_list); static void kvm_mmu_commit_zap_page(struct kvm *kvm, struct list_head *invalid_list); -/* - * NOTE: we should pay more attention on the zapped-obsolete page - * (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk - * since it has been deleted from active_mmu_pages but still can be found - * at hast list. - * - * for_each_valid_sp() has skipped that kind of pages. - */ #define for_each_valid_sp(_kvm, _sp, _gfn) \ hlist_for_each_entry(_sp, \ &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \ - if (is_obsolete_sp((_kvm), (_sp)) || (_sp)->role.invalid) { \ + if ((_sp)->role.invalid) { \ } else #define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn) \ @@ -2231,18 +2228,28 @@ static bool __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, return true; } +static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm, + struct list_head *invalid_list, + bool remote_flush) +{ + if (!remote_flush && !list_empty(invalid_list)) + return false; + + if (!list_empty(invalid_list)) + kvm_mmu_commit_zap_page(kvm, invalid_list); + else + kvm_flush_remote_tlbs(kvm); + return true; +} + static void kvm_mmu_flush_or_zap(struct kvm_vcpu *vcpu, struct list_head *invalid_list, bool remote_flush, bool local_flush) { - if (!list_empty(invalid_list)) { - kvm_mmu_commit_zap_page(vcpu->kvm, invalid_list); + if (kvm_mmu_remote_flush_or_zap(vcpu->kvm, invalid_list, remote_flush)) return; - } - if (remote_flush) - kvm_flush_remote_tlbs(vcpu->kvm); - else if (local_flush) + if (local_flush) kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); } @@ -2253,11 +2260,6 @@ static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { } static void mmu_audit_disable(void) { } #endif -static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp) -{ - return unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen); -} - static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, struct list_head *invalid_list) { @@ -2482,7 +2484,6 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, if (level > PT_PAGE_TABLE_LEVEL && need_sync) flush |= kvm_sync_pages(vcpu, gfn, &invalid_list); } - sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen; clear_page(sp->spt); trace_kvm_mmu_get_page(sp, true); @@ -2668,17 +2669,22 @@ static int mmu_zap_unsync_children(struct kvm *kvm, return zapped; } -static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, - struct list_head *invalid_list) +static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm, + struct kvm_mmu_page *sp, + struct list_head *invalid_list, + int *nr_zapped) { - int ret; + bool list_unstable; trace_kvm_mmu_prepare_zap_page(sp); ++kvm->stat.mmu_shadow_zapped; - ret = mmu_zap_unsync_children(kvm, sp, invalid_list); + *nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list); kvm_mmu_page_unlink_children(kvm, sp); kvm_mmu_unlink_parents(kvm, sp); + /* Zapping children means active_mmu_pages has become unstable. */ + list_unstable = *nr_zapped; + if (!sp->role.invalid && !sp->role.direct) unaccount_shadowed(kvm, sp); @@ -2686,22 +2692,27 @@ static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, kvm_unlink_unsync_page(kvm, sp); if (!sp->root_count) { /* Count self */ - ret++; + (*nr_zapped)++; list_move(&sp->link, invalid_list); kvm_mod_used_mmu_pages(kvm, -1); } else { list_move(&sp->link, &kvm->arch.active_mmu_pages); - /* - * The obsolete pages can not be used on any vcpus. - * See the comments in kvm_mmu_invalidate_zap_all_pages(). - */ - if (!sp->role.invalid && !is_obsolete_sp(kvm, sp)) + if (!sp->role.invalid) kvm_reload_remote_mmus(kvm); } sp->role.invalid = 1; - return ret; + return list_unstable; +} + +static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, + struct list_head *invalid_list) +{ + int nr_zapped; + + __kvm_mmu_prepare_zap_page(kvm, sp, invalid_list, &nr_zapped); + return nr_zapped; } static void kvm_mmu_commit_zap_page(struct kvm *kvm, @@ -3703,7 +3714,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) u64 *lm_root; - lm_root = (void*)get_zeroed_page(GFP_KERNEL); + lm_root = (void*)get_zeroed_page(GFP_KERNEL_ACCOUNT); if (lm_root == NULL) return 1; @@ -4204,14 +4215,6 @@ static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3, return false; if (cached_root_available(vcpu, new_cr3, new_role)) { - /* - * It is possible that the cached previous root page is - * obsolete because of a change in the MMU - * generation number. However, that is accompanied by - * KVM_REQ_MMU_RELOAD, which will free the root that we - * have set here and allocate a new one. - */ - kvm_make_request(KVM_REQ_LOAD_CR3, vcpu); if (!skip_tlb_flush) { kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); @@ -5486,6 +5489,76 @@ void kvm_disable_tdp(void) } EXPORT_SYMBOL_GPL(kvm_disable_tdp); + +/* The return value indicates if tlb flush on all vcpus is needed. */ +typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head); + +/* The caller should hold mmu-lock before calling this function. */ +static __always_inline bool +slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot, + slot_level_handler fn, int start_level, int end_level, + gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb) +{ + struct slot_rmap_walk_iterator iterator; + bool flush = false; + + for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn, + end_gfn, &iterator) { + if (iterator.rmap) + flush |= fn(kvm, iterator.rmap); + + if (need_resched() || spin_needbreak(&kvm->mmu_lock)) { + if (flush && lock_flush_tlb) { + kvm_flush_remote_tlbs(kvm); + flush = false; + } + cond_resched_lock(&kvm->mmu_lock); + } + } + + if (flush && lock_flush_tlb) { + kvm_flush_remote_tlbs(kvm); + flush = false; + } + + return flush; +} + +static __always_inline bool +slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot, + slot_level_handler fn, int start_level, int end_level, + bool lock_flush_tlb) +{ + return slot_handle_level_range(kvm, memslot, fn, start_level, + end_level, memslot->base_gfn, + memslot->base_gfn + memslot->npages - 1, + lock_flush_tlb); +} + +static __always_inline bool +slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot, + slot_level_handler fn, bool lock_flush_tlb) +{ + return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL, + PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); +} + +static __always_inline bool +slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot, + slot_level_handler fn, bool lock_flush_tlb) +{ + return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL + 1, + PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); +} + +static __always_inline bool +slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot, + slot_level_handler fn, bool lock_flush_tlb) +{ + return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL, + PT_PAGE_TABLE_LEVEL, lock_flush_tlb); +} + static void free_mmu_pages(struct kvm_vcpu *vcpu) { free_page((unsigned long)vcpu->arch.mmu->pae_root); @@ -5505,7 +5578,7 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) * Therefore we need to allocate shadow page tables in the first * 4GB of memory, which happens to fit the DMA32 zone. */ - page = alloc_page(GFP_KERNEL | __GFP_DMA32); + page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32); if (!page) return -ENOMEM; @@ -5543,105 +5616,62 @@ static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, struct kvm_page_track_notifier_node *node) { - kvm_mmu_invalidate_zap_all_pages(kvm); -} - -void kvm_mmu_init_vm(struct kvm *kvm) -{ - struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; - - node->track_write = kvm_mmu_pte_write; - node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot; - kvm_page_track_register_notifier(kvm, node); -} + struct kvm_mmu_page *sp; + LIST_HEAD(invalid_list); + unsigned long i; + bool flush; + gfn_t gfn; -void kvm_mmu_uninit_vm(struct kvm *kvm) -{ - struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; + spin_lock(&kvm->mmu_lock); - kvm_page_track_unregister_notifier(kvm, node); -} + if (list_empty(&kvm->arch.active_mmu_pages)) + goto out_unlock; -/* The return value indicates if tlb flush on all vcpus is needed. */ -typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head); + flush = slot_handle_all_level(kvm, slot, kvm_zap_rmapp, false); -/* The caller should hold mmu-lock before calling this function. */ -static __always_inline bool -slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot, - slot_level_handler fn, int start_level, int end_level, - gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb) -{ - struct slot_rmap_walk_iterator iterator; - bool flush = false; + for (i = 0; i < slot->npages; i++) { + gfn = slot->base_gfn + i; - for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn, - end_gfn, &iterator) { - if (iterator.rmap) - flush |= fn(kvm, iterator.rmap); + for_each_valid_sp(kvm, sp, gfn) { + if (sp->gfn != gfn) + continue; + kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list); + } if (need_resched() || spin_needbreak(&kvm->mmu_lock)) { - if (flush && lock_flush_tlb) { - kvm_flush_remote_tlbs(kvm); - flush = false; - } + kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush); + flush = false; cond_resched_lock(&kvm->mmu_lock); } } + kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush); - if (flush && lock_flush_tlb) { - kvm_flush_remote_tlbs(kvm); - flush = false; - } - - return flush; +out_unlock: + spin_unlock(&kvm->mmu_lock); } -static __always_inline bool -slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot, - slot_level_handler fn, int start_level, int end_level, - bool lock_flush_tlb) +void kvm_mmu_init_vm(struct kvm *kvm) { - return slot_handle_level_range(kvm, memslot, fn, start_level, - end_level, memslot->base_gfn, - memslot->base_gfn + memslot->npages - 1, - lock_flush_tlb); -} + struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; -static __always_inline bool -slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot, - slot_level_handler fn, bool lock_flush_tlb) -{ - return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL, - PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); + node->track_write = kvm_mmu_pte_write; + node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot; + kvm_page_track_register_notifier(kvm, node); } -static __always_inline bool -slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot, - slot_level_handler fn, bool lock_flush_tlb) +void kvm_mmu_uninit_vm(struct kvm *kvm) { - return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL + 1, - PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); -} + struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; -static __always_inline bool -slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot, - slot_level_handler fn, bool lock_flush_tlb) -{ - return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL, - PT_PAGE_TABLE_LEVEL, lock_flush_tlb); + kvm_page_track_unregister_notifier(kvm, node); } void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) { struct kvm_memslots *slots; struct kvm_memory_slot *memslot; - bool flush_tlb = true; - bool flush = false; int i; - if (kvm_available_flush_tlb_with_range()) - flush_tlb = false; - spin_lock(&kvm->mmu_lock); for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { slots = __kvm_memslots(kvm, i); @@ -5653,17 +5683,12 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) if (start >= end) continue; - flush |= slot_handle_level_range(kvm, memslot, - kvm_zap_rmapp, PT_PAGE_TABLE_LEVEL, - PT_MAX_HUGEPAGE_LEVEL, start, - end - 1, flush_tlb); + slot_handle_level_range(kvm, memslot, kvm_zap_rmapp, + PT_PAGE_TABLE_LEVEL, PT_MAX_HUGEPAGE_LEVEL, + start, end - 1, true); } } - if (flush) - kvm_flush_remote_tlbs_with_address(kvm, gfn_start, - gfn_end - gfn_start + 1); - spin_unlock(&kvm->mmu_lock); } @@ -5815,101 +5840,58 @@ void kvm_mmu_slot_set_dirty(struct kvm *kvm, } EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty); -#define BATCH_ZAP_PAGES 10 -static void kvm_zap_obsolete_pages(struct kvm *kvm) +static void __kvm_mmu_zap_all(struct kvm *kvm, bool mmio_only) { struct kvm_mmu_page *sp, *node; - int batch = 0; + LIST_HEAD(invalid_list); + int ign; + spin_lock(&kvm->mmu_lock); restart: - list_for_each_entry_safe_reverse(sp, node, - &kvm->arch.active_mmu_pages, link) { - int ret; - - /* - * No obsolete page exists before new created page since - * active_mmu_pages is the FIFO list. - */ - if (!is_obsolete_sp(kvm, sp)) - break; - - /* - * Since we are reversely walking the list and the invalid - * list will be moved to the head, skip the invalid page - * can help us to avoid the infinity list walking. - */ - if (sp->role.invalid) + list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) { + if (mmio_only && !sp->mmio_cached) continue; - - /* - * Need not flush tlb since we only zap the sp with invalid - * generation number. - */ - if (batch >= BATCH_ZAP_PAGES && - cond_resched_lock(&kvm->mmu_lock)) { - batch = 0; + if (sp->role.invalid && sp->root_count) + continue; + if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign)) { + WARN_ON_ONCE(mmio_only); goto restart; } - - ret = kvm_mmu_prepare_zap_page(kvm, sp, - &kvm->arch.zapped_obsolete_pages); - batch += ret; - - if (ret) + if (cond_resched_lock(&kvm->mmu_lock)) goto restart; } - /* - * Should flush tlb before free page tables since lockless-walking - * may use the pages. - */ - kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages); -} - -/* - * Fast invalidate all shadow pages and use lock-break technique - * to zap obsolete pages. - * - * It's required when memslot is being deleted or VM is being - * destroyed, in these cases, we should ensure that KVM MMU does - * not use any resource of the being-deleted slot or all slots - * after calling the function. - */ -void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm) -{ - spin_lock(&kvm->mmu_lock); - trace_kvm_mmu_invalidate_zap_all_pages(kvm); - kvm->arch.mmu_valid_gen++; - - /* - * Notify all vcpus to reload its shadow page table - * and flush TLB. Then all vcpus will switch to new - * shadow page table with the new mmu_valid_gen. - * - * Note: we should do this under the protection of - * mmu-lock, otherwise, vcpu would purge shadow page - * but miss tlb flush. - */ - kvm_reload_remote_mmus(kvm); - - kvm_zap_obsolete_pages(kvm); + kvm_mmu_commit_zap_page(kvm, &invalid_list); spin_unlock(&kvm->mmu_lock); } -static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm) +void kvm_mmu_zap_all(struct kvm *kvm) { - return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages)); + return __kvm_mmu_zap_all(kvm, false); } -void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots) +void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) { + WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS); + + gen &= MMIO_SPTE_GEN_MASK; + /* - * The very rare case: if the generation-number is round, + * Generation numbers are incremented in multiples of the number of + * address spaces in order to provide unique generations across all + * address spaces. Strip what is effectively the address space + * modifier prior to checking for a wrap of the MMIO generation so + * that a wrap in any address space is detected. + */ + gen &= ~((u64)KVM_ADDRESS_SPACE_NUM - 1); + + /* + * The very rare case: if the MMIO generation number has wrapped, * zap all shadow pages. */ - if (unlikely((slots->generation & MMIO_GEN_MASK) == 0)) { + if (unlikely(gen == 0)) { kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n"); - kvm_mmu_invalidate_zap_all_pages(kvm); + __kvm_mmu_zap_all(kvm, true); } } @@ -5940,24 +5922,16 @@ mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) * want to shrink a VM that only started to populate its MMU * anyway. */ - if (!kvm->arch.n_used_mmu_pages && - !kvm_has_zapped_obsolete_pages(kvm)) + if (!kvm->arch.n_used_mmu_pages) continue; idx = srcu_read_lock(&kvm->srcu); spin_lock(&kvm->mmu_lock); - if (kvm_has_zapped_obsolete_pages(kvm)) { - kvm_mmu_commit_zap_page(kvm, - &kvm->arch.zapped_obsolete_pages); - goto unlock; - } - if (prepare_zap_oldest_mmu_page(kvm, &invalid_list)) freed++; kvm_mmu_commit_zap_page(kvm, &invalid_list); -unlock: spin_unlock(&kvm->mmu_lock); srcu_read_unlock(&kvm->srcu, idx); diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index c7b333147c4a6..bbdc60f2fae89 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -203,7 +203,6 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, return -(u32)fault & errcode; } -void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm); void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end); void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn); diff --git a/arch/x86/kvm/mmutrace.h b/arch/x86/kvm/mmutrace.h index c73bf4e4988cb..9f6c855a00439 100644 --- a/arch/x86/kvm/mmutrace.h +++ b/arch/x86/kvm/mmutrace.h @@ -8,18 +8,16 @@ #undef TRACE_SYSTEM #define TRACE_SYSTEM kvmmmu -#define KVM_MMU_PAGE_FIELDS \ - __field(unsigned long, mmu_valid_gen) \ - __field(__u64, gfn) \ - __field(__u32, role) \ - __field(__u32, root_count) \ +#define KVM_MMU_PAGE_FIELDS \ + __field(__u64, gfn) \ + __field(__u32, role) \ + __field(__u32, root_count) \ __field(bool, unsync) -#define KVM_MMU_PAGE_ASSIGN(sp) \ - __entry->mmu_valid_gen = sp->mmu_valid_gen; \ - __entry->gfn = sp->gfn; \ - __entry->role = sp->role.word; \ - __entry->root_count = sp->root_count; \ +#define KVM_MMU_PAGE_ASSIGN(sp) \ + __entry->gfn = sp->gfn; \ + __entry->role = sp->role.word; \ + __entry->root_count = sp->root_count; \ __entry->unsync = sp->unsync; #define KVM_MMU_PAGE_PRINTK() ({ \ @@ -31,9 +29,8 @@ \ role.word = __entry->role; \ \ - trace_seq_printf(p, "sp gen %lx gfn %llx l%u%s q%u%s %s%s" \ + trace_seq_printf(p, "sp gfn %llx l%u%s q%u%s %s%s" \ " %snxe %sad root %u %s%c", \ - __entry->mmu_valid_gen, \ __entry->gfn, role.level, \ role.cr4_pae ? " pae" : "", \ role.quadrant, \ @@ -283,27 +280,6 @@ TRACE_EVENT( ); TRACE_EVENT( - kvm_mmu_invalidate_zap_all_pages, - TP_PROTO(struct kvm *kvm), - TP_ARGS(kvm), - - TP_STRUCT__entry( - __field(unsigned long, mmu_valid_gen) - __field(unsigned int, mmu_used_pages) - ), - - TP_fast_assign( - __entry->mmu_valid_gen = kvm->arch.mmu_valid_gen; - __entry->mmu_used_pages = kvm->arch.n_used_mmu_pages; - ), - - TP_printk("kvm-mmu-valid-gen %lx used_pages %x", - __entry->mmu_valid_gen, __entry->mmu_used_pages - ) -); - - -TRACE_EVENT( check_mmio_spte, TP_PROTO(u64 spte, unsigned int kvm_gen, unsigned int spte_gen), TP_ARGS(spte, kvm_gen, spte_gen), diff --git a/arch/x86/kvm/page_track.c b/arch/x86/kvm/page_track.c index 3052a59a30655..fd04d462fdaee 100644 --- a/arch/x86/kvm/page_track.c +++ b/arch/x86/kvm/page_track.c @@ -42,7 +42,7 @@ int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { slot->arch.gfn_track[i] = kvcalloc(npages, sizeof(*slot->arch.gfn_track[i]), - GFP_KERNEL); + GFP_KERNEL_ACCOUNT); if (!slot->arch.gfn_track[i]) goto track_free; } diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index f13a3a24d3609..b5b128a0a0512 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -145,7 +145,6 @@ struct kvm_svm { /* Struct members for AVIC */ u32 avic_vm_id; - u32 ldr_mode; struct page *avic_logical_id_table_page; struct page *avic_physical_id_table_page; struct hlist_node hnode; @@ -236,6 +235,7 @@ struct vcpu_svm { bool nrips_enabled : 1; u32 ldr_reg; + u32 dfr_reg; struct page *avic_backing_page; u64 *avic_physical_id_cache; bool avic_is_running; @@ -1795,9 +1795,10 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, /* Avoid using vmalloc for smaller buffers. */ size = npages * sizeof(struct page *); if (size > PAGE_SIZE) - pages = vmalloc(size); + pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO, + PAGE_KERNEL); else - pages = kmalloc(size, GFP_KERNEL); + pages = kmalloc(size, GFP_KERNEL_ACCOUNT); if (!pages) return NULL; @@ -1865,7 +1866,9 @@ static void __unregister_enc_region_locked(struct kvm *kvm, static struct kvm *svm_vm_alloc(void) { - struct kvm_svm *kvm_svm = vzalloc(sizeof(struct kvm_svm)); + struct kvm_svm *kvm_svm = __vmalloc(sizeof(struct kvm_svm), + GFP_KERNEL_ACCOUNT | __GFP_ZERO, + PAGE_KERNEL); return &kvm_svm->kvm; } @@ -1940,7 +1943,7 @@ static int avic_vm_init(struct kvm *kvm) return 0; /* Allocating physical APIC ID table (4KB) */ - p_page = alloc_page(GFP_KERNEL); + p_page = alloc_page(GFP_KERNEL_ACCOUNT); if (!p_page) goto free_avic; @@ -1948,7 +1951,7 @@ static int avic_vm_init(struct kvm *kvm) clear_page(page_address(p_page)); /* Allocating logical APIC ID table (4KB) */ - l_page = alloc_page(GFP_KERNEL); + l_page = alloc_page(GFP_KERNEL_ACCOUNT); if (!l_page) goto free_avic; @@ -2106,6 +2109,7 @@ static int avic_init_vcpu(struct vcpu_svm *svm) INIT_LIST_HEAD(&svm->ir_list); spin_lock_init(&svm->ir_list_lock); + svm->dfr_reg = APIC_DFR_FLAT; return ret; } @@ -2119,13 +2123,14 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) struct page *nested_msrpm_pages; int err; - svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); + svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT); if (!svm) { err = -ENOMEM; goto out; } - svm->vcpu.arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache, GFP_KERNEL); + svm->vcpu.arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache, + GFP_KERNEL_ACCOUNT); if (!svm->vcpu.arch.guest_fpu) { printk(KERN_ERR "kvm: failed to allocate vcpu's fpu\n"); err = -ENOMEM; @@ -2137,19 +2142,19 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) goto free_svm; err = -ENOMEM; - page = alloc_page(GFP_KERNEL); + page = alloc_page(GFP_KERNEL_ACCOUNT); if (!page) goto uninit; - msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER); + msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER); if (!msrpm_pages) goto free_page1; - nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER); + nested_msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER); if (!nested_msrpm_pages) goto free_page2; - hsave_page = alloc_page(GFP_KERNEL); + hsave_page = alloc_page(GFP_KERNEL_ACCOUNT); if (!hsave_page) goto free_page3; @@ -4565,8 +4570,7 @@ static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) return &logical_apic_id_table[index]; } -static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr, - bool valid) +static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr) { bool flat; u32 *entry, new_entry; @@ -4579,31 +4583,39 @@ static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr, new_entry = READ_ONCE(*entry); new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK); - if (valid) - new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; - else - new_entry &= ~AVIC_LOGICAL_ID_ENTRY_VALID_MASK; + new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; WRITE_ONCE(*entry, new_entry); return 0; } +static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + bool flat = svm->dfr_reg == APIC_DFR_FLAT; + u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat); + + if (entry) + WRITE_ONCE(*entry, (u32) ~AVIC_LOGICAL_ID_ENTRY_VALID_MASK); +} + static int avic_handle_ldr_update(struct kvm_vcpu *vcpu) { - int ret; + int ret = 0; struct vcpu_svm *svm = to_svm(vcpu); u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR); - if (!ldr) - return 1; + if (ldr == svm->ldr_reg) + return 0; - ret = avic_ldr_write(vcpu, vcpu->vcpu_id, ldr, true); - if (ret && svm->ldr_reg) { - avic_ldr_write(vcpu, 0, svm->ldr_reg, false); - svm->ldr_reg = 0; - } else { + avic_invalidate_logical_id_entry(vcpu); + + if (ldr) + ret = avic_ldr_write(vcpu, vcpu->vcpu_id, ldr); + + if (!ret) svm->ldr_reg = ldr; - } + return ret; } @@ -4637,27 +4649,16 @@ static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu) return 0; } -static int avic_handle_dfr_update(struct kvm_vcpu *vcpu) +static void avic_handle_dfr_update(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR); - u32 mod = (dfr >> 28) & 0xf; - /* - * We assume that all local APICs are using the same type. - * If this changes, we need to flush the AVIC logical - * APID id table. - */ - if (kvm_svm->ldr_mode == mod) - return 0; - - clear_page(page_address(kvm_svm->avic_logical_id_table_page)); - kvm_svm->ldr_mode = mod; + if (svm->dfr_reg == dfr) + return; - if (svm->ldr_reg) - avic_handle_ldr_update(vcpu); - return 0; + avic_invalidate_logical_id_entry(vcpu); + svm->dfr_reg = dfr; } static int avic_unaccel_trap_write(struct vcpu_svm *svm) @@ -5125,11 +5126,11 @@ static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb = svm->vmcb; - if (!kvm_vcpu_apicv_active(&svm->vcpu)) - return; - - vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; - mark_dirty(vmcb, VMCB_INTR); + if (kvm_vcpu_apicv_active(vcpu)) + vmcb->control.int_ctl |= AVIC_ENABLE_MASK; + else + vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; + mark_dirty(vmcb, VMCB_AVIC); } static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) @@ -5195,7 +5196,7 @@ static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) * Allocating new amd_iommu_pi_data, which will get * add to the per-vcpu ir_list. */ - ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL); + ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT); if (!ir) { ret = -ENOMEM; goto out; @@ -6163,8 +6164,7 @@ static inline void avic_post_state_restore(struct kvm_vcpu *vcpu) { if (avic_handle_apic_id_update(vcpu) != 0) return; - if (avic_handle_dfr_update(vcpu) != 0) - return; + avic_handle_dfr_update(vcpu); avic_handle_ldr_update(vcpu); } @@ -6311,7 +6311,7 @@ static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error) if (ret) return ret; - data = kzalloc(sizeof(*data), GFP_KERNEL); + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); if (!data) return -ENOMEM; @@ -6361,7 +6361,7 @@ static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) return -EFAULT; - start = kzalloc(sizeof(*start), GFP_KERNEL); + start = kzalloc(sizeof(*start), GFP_KERNEL_ACCOUNT); if (!start) return -ENOMEM; @@ -6458,7 +6458,7 @@ static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) return -EFAULT; - data = kzalloc(sizeof(*data), GFP_KERNEL); + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); if (!data) return -ENOMEM; @@ -6535,7 +6535,7 @@ static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp) if (copy_from_user(¶ms, measure, sizeof(params))) return -EFAULT; - data = kzalloc(sizeof(*data), GFP_KERNEL); + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); if (!data) return -ENOMEM; @@ -6597,7 +6597,7 @@ static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) if (!sev_guest(kvm)) return -ENOTTY; - data = kzalloc(sizeof(*data), GFP_KERNEL); + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); if (!data) return -ENOMEM; @@ -6618,7 +6618,7 @@ static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp) if (!sev_guest(kvm)) return -ENOTTY; - data = kzalloc(sizeof(*data), GFP_KERNEL); + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); if (!data) return -ENOMEM; @@ -6646,7 +6646,7 @@ static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src, struct sev_data_dbg *data; int ret; - data = kzalloc(sizeof(*data), GFP_KERNEL); + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); if (!data) return -ENOMEM; @@ -6901,7 +6901,7 @@ static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp) } ret = -ENOMEM; - data = kzalloc(sizeof(*data), GFP_KERNEL); + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); if (!data) goto e_unpin_memory; @@ -7007,7 +7007,7 @@ static int svm_register_enc_region(struct kvm *kvm, if (range->addr > ULONG_MAX || range->size > ULONG_MAX) return -EINVAL; - region = kzalloc(sizeof(*region), GFP_KERNEL); + region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT); if (!region) return -ENOMEM; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index d737a51a53ca3..f24a2c2250706 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -211,7 +211,6 @@ static void free_nested(struct kvm_vcpu *vcpu) if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) return; - hrtimer_cancel(&vmx->nested.preemption_timer); vmx->nested.vmxon = false; vmx->nested.smm.vmxon = false; free_vpid(vmx->nested.vpid02); @@ -274,6 +273,7 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) { vcpu_load(vcpu); + vmx_leave_nested(vcpu); vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01); free_nested(vcpu); vcpu_put(vcpu); @@ -1980,17 +1980,6 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) prepare_vmcs02_early_full(vmx, vmcs12); /* - * HOST_RSP is normally set correctly in vmx_vcpu_run() just before - * entry, but only if the current (host) sp changed from the value - * we wrote last (vmx->host_rsp). This cache is no longer relevant - * if we switch vmcs, and rather than hold a separate cache per vmcs, - * here we just force the write to happen on entry. host_rsp will - * also be written unconditionally by nested_vmx_check_vmentry_hw() - * if we are doing early consistency checks via hardware. - */ - vmx->host_rsp = 0; - - /* * PIN CONTROLS */ exec_control = vmcs12->pin_based_vm_exec_control; @@ -2289,10 +2278,6 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, } vmx_set_rflags(vcpu, vmcs12->guest_rflags); - vmx->nested.preemption_timer_expired = false; - if (nested_cpu_has_preemption_timer(vmcs12)) - vmx_start_preemption_timer(vcpu); - /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the * bitwise-or of what L1 wants to trap for L2, and what we want to * trap. Note that CR0.TS also needs updating - we do this later. @@ -2722,6 +2707,7 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); unsigned long cr3, cr4; + bool vm_fail; if (!nested_early_check) return 0; @@ -2755,29 +2741,34 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) vmx->loaded_vmcs->host_state.cr4 = cr4; } - vmx->__launched = vmx->loaded_vmcs->launched; - asm( - /* Set HOST_RSP */ "sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */ - __ex("vmwrite %%" _ASM_SP ", %%" _ASM_DX) "\n\t" - "mov %%" _ASM_SP ", %c[host_rsp](%1)\n\t" + "cmp %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t" + "je 1f \n\t" + __ex("vmwrite %%" _ASM_SP ", %[HOST_RSP]") "\n\t" + "mov %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t" + "1: \n\t" "add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */ /* Check if vmlaunch or vmresume is needed */ - "cmpl $0, %c[launched](%% " _ASM_CX")\n\t" + "cmpb $0, %c[launched](%[loaded_vmcs])\n\t" + /* + * VMLAUNCH and VMRESUME clear RFLAGS.{CF,ZF} on VM-Exit, set + * RFLAGS.CF on VM-Fail Invalid and set RFLAGS.ZF on VM-Fail + * Valid. vmx_vmenter() directly "returns" RFLAGS, and so the + * results of VM-Enter is captured via CC_{SET,OUT} to vm_fail. + */ "call vmx_vmenter\n\t" - /* Set vmx->fail accordingly */ - "setbe %c[fail](%% " _ASM_CX")\n\t" - : ASM_CALL_CONSTRAINT - : "c"(vmx), "d"((unsigned long)HOST_RSP), - [launched]"i"(offsetof(struct vcpu_vmx, __launched)), - [fail]"i"(offsetof(struct vcpu_vmx, fail)), - [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)), + CC_SET(be) + : ASM_CALL_CONSTRAINT, CC_OUT(be) (vm_fail) + : [HOST_RSP]"r"((unsigned long)HOST_RSP), + [loaded_vmcs]"r"(vmx->loaded_vmcs), + [launched]"i"(offsetof(struct loaded_vmcs, launched)), + [host_state_rsp]"i"(offsetof(struct loaded_vmcs, host_state.rsp)), [wordsize]"i"(sizeof(ulong)) - : "rax", "cc", "memory" + : "cc", "memory" ); preempt_enable(); @@ -2787,10 +2778,9 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) if (vmx->msr_autoload.guest.nr) vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); - if (vmx->fail) { + if (vm_fail) { WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != VMXERR_ENTRY_INVALID_CONTROL_FIELD); - vmx->fail = 0; return 1; } @@ -2813,8 +2803,6 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) return 0; } -STACK_FRAME_NON_STANDARD(nested_vmx_check_vmentry_hw); - static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12); @@ -3031,6 +3019,15 @@ int nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry) kvm_make_request(KVM_REQ_EVENT, vcpu); /* + * Do not start the preemption timer hrtimer until after we know + * we are successful, so that only nested_vmx_vmexit needs to cancel + * the timer. + */ + vmx->nested.preemption_timer_expired = false; + if (nested_cpu_has_preemption_timer(vmcs12)) + vmx_start_preemption_timer(vcpu); + + /* * Note no nested_vmx_succeed or nested_vmx_fail here. At this point * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet * returned as far as L1 is concerned. It will only return (and set @@ -3450,13 +3447,10 @@ static void sync_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) else vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; - if (nested_cpu_has_preemption_timer(vmcs12)) { - if (vmcs12->vm_exit_controls & - VM_EXIT_SAVE_VMX_PREEMPTION_TIMER) + if (nested_cpu_has_preemption_timer(vmcs12) && + vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER) vmcs12->vmx_preemption_timer_value = vmx_get_preemption_timer_value(vcpu); - hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); - } /* * In some cases (usually, nested EPT), L2 is allowed to change its @@ -3864,6 +3858,9 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, leave_guest_mode(vcpu); + if (nested_cpu_has_preemption_timer(vmcs12)) + hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); + if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) vcpu->arch.tsc_offset -= vmcs12->tsc_offset; @@ -3915,9 +3912,6 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, vmx_flush_tlb(vcpu, true); } - /* This is needed for same reason as it was needed in prepare_vmcs02 */ - vmx->host_rsp = 0; - /* Unpin physical memory we referred to in vmcs02 */ if (vmx->nested.apic_access_page) { kvm_release_page_dirty(vmx->nested.apic_access_page); @@ -4035,25 +4029,50 @@ int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, /* Addr = segment_base + offset */ /* offset = base + [index * scale] + displacement */ off = exit_qualification; /* holds the displacement */ + if (addr_size == 1) + off = (gva_t)sign_extend64(off, 31); + else if (addr_size == 0) + off = (gva_t)sign_extend64(off, 15); if (base_is_valid) off += kvm_register_read(vcpu, base_reg); if (index_is_valid) off += kvm_register_read(vcpu, index_reg)<<scaling; vmx_get_segment(vcpu, &s, seg_reg); - *ret = s.base + off; + /* + * The effective address, i.e. @off, of a memory operand is truncated + * based on the address size of the instruction. Note that this is + * the *effective address*, i.e. the address prior to accounting for + * the segment's base. + */ if (addr_size == 1) /* 32 bit */ - *ret &= 0xffffffff; + off &= 0xffffffff; + else if (addr_size == 0) /* 16 bit */ + off &= 0xffff; /* Checks for #GP/#SS exceptions. */ exn = false; if (is_long_mode(vcpu)) { + /* + * The virtual/linear address is never truncated in 64-bit + * mode, e.g. a 32-bit address size can yield a 64-bit virtual + * address when using FS/GS with a non-zero base. + */ + *ret = s.base + off; + /* Long mode: #GP(0)/#SS(0) if the memory address is in a * non-canonical form. This is the only check on the memory * destination for long mode! */ exn = is_noncanonical_address(*ret, vcpu); - } else if (is_protmode(vcpu)) { + } else { + /* + * When not in long mode, the virtual/linear address is + * unconditionally truncated to 32 bits regardless of the + * address size. + */ + *ret = (s.base + off) & 0xffffffff; + /* Protected mode: apply checks for segment validity in the * following order: * - segment type check (#GP(0) may be thrown) @@ -4077,10 +4096,16 @@ int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. */ exn = (s.unusable != 0); - /* Protected mode: #GP(0)/#SS(0) if the memory - * operand is outside the segment limit. + + /* + * Protected mode: #GP(0)/#SS(0) if the memory operand is + * outside the segment limit. All CPUs that support VMX ignore + * limit checks for flat segments, i.e. segments with base==0, + * limit==0xffffffff and of type expand-up data or code. */ - exn = exn || (off + sizeof(u64) > s.limit); + if (!(s.base == 0 && s.limit == 0xffffffff && + ((s.type & 8) || !(s.type & 4)))) + exn = exn || (off + sizeof(u64) > s.limit); } if (exn) { kvm_queue_exception_e(vcpu, @@ -4145,11 +4170,11 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu) if (r < 0) goto out_vmcs02; - vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL); + vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); if (!vmx->nested.cached_vmcs12) goto out_cached_vmcs12; - vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL); + vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); if (!vmx->nested.cached_shadow_vmcs12) goto out_cached_shadow_vmcs12; @@ -5696,6 +5721,10 @@ __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) enable_shadow_vmcs = 0; if (enable_shadow_vmcs) { for (i = 0; i < VMX_BITMAP_NR; i++) { + /* + * The vmx_bitmap is not tied to a VM and so should + * not be charged to a memcg. + */ vmx_bitmap[i] = (unsigned long *) __get_free_page(GFP_KERNEL); if (!vmx_bitmap[i]) { diff --git a/arch/x86/kvm/vmx/vmcs.h b/arch/x86/kvm/vmx/vmcs.h index 6def3ba88e3b3..cb6079f8a227f 100644 --- a/arch/x86/kvm/vmx/vmcs.h +++ b/arch/x86/kvm/vmx/vmcs.h @@ -34,6 +34,7 @@ struct vmcs_host_state { unsigned long cr4; /* May not match real cr4 */ unsigned long gs_base; unsigned long fs_base; + unsigned long rsp; u16 fs_sel, gs_sel, ldt_sel; #ifdef CONFIG_X86_64 diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S index bcef2c7e9bc48..7b272738c5768 100644 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -1,6 +1,30 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/asm.h> +#include <asm/bitsperlong.h> +#include <asm/kvm_vcpu_regs.h> + +#define WORD_SIZE (BITS_PER_LONG / 8) + +#define VCPU_RAX __VCPU_REGS_RAX * WORD_SIZE +#define VCPU_RCX __VCPU_REGS_RCX * WORD_SIZE +#define VCPU_RDX __VCPU_REGS_RDX * WORD_SIZE +#define VCPU_RBX __VCPU_REGS_RBX * WORD_SIZE +/* Intentionally omit RSP as it's context switched by hardware */ +#define VCPU_RBP __VCPU_REGS_RBP * WORD_SIZE +#define VCPU_RSI __VCPU_REGS_RSI * WORD_SIZE +#define VCPU_RDI __VCPU_REGS_RDI * WORD_SIZE + +#ifdef CONFIG_X86_64 +#define VCPU_R8 __VCPU_REGS_R8 * WORD_SIZE +#define VCPU_R9 __VCPU_REGS_R9 * WORD_SIZE +#define VCPU_R10 __VCPU_REGS_R10 * WORD_SIZE +#define VCPU_R11 __VCPU_REGS_R11 * WORD_SIZE +#define VCPU_R12 __VCPU_REGS_R12 * WORD_SIZE +#define VCPU_R13 __VCPU_REGS_R13 * WORD_SIZE +#define VCPU_R14 __VCPU_REGS_R14 * WORD_SIZE +#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE +#endif .text @@ -55,3 +79,146 @@ ENDPROC(vmx_vmenter) ENTRY(vmx_vmexit) ret ENDPROC(vmx_vmexit) + +/** + * __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode + * @vmx: struct vcpu_vmx * + * @regs: unsigned long * (to guest registers) + * @launched: %true if the VMCS has been launched + * + * Returns: + * 0 on VM-Exit, 1 on VM-Fail + */ +ENTRY(__vmx_vcpu_run) + push %_ASM_BP + mov %_ASM_SP, %_ASM_BP +#ifdef CONFIG_X86_64 + push %r15 + push %r14 + push %r13 + push %r12 +#else + push %edi + push %esi +#endif + push %_ASM_BX + + /* + * Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and + * @regs is needed after VM-Exit to save the guest's register values. + */ + push %_ASM_ARG2 + + /* Copy @launched to BL, _ASM_ARG3 is volatile. */ + mov %_ASM_ARG3B, %bl + + /* Adjust RSP to account for the CALL to vmx_vmenter(). */ + lea -WORD_SIZE(%_ASM_SP), %_ASM_ARG2 + call vmx_update_host_rsp + + /* Load @regs to RAX. */ + mov (%_ASM_SP), %_ASM_AX + + /* Check if vmlaunch or vmresume is needed */ + cmpb $0, %bl + + /* Load guest registers. Don't clobber flags. */ + mov VCPU_RBX(%_ASM_AX), %_ASM_BX + mov VCPU_RCX(%_ASM_AX), %_ASM_CX + mov VCPU_RDX(%_ASM_AX), %_ASM_DX + mov VCPU_RSI(%_ASM_AX), %_ASM_SI + mov VCPU_RDI(%_ASM_AX), %_ASM_DI + mov VCPU_RBP(%_ASM_AX), %_ASM_BP +#ifdef CONFIG_X86_64 + mov VCPU_R8 (%_ASM_AX), %r8 + mov VCPU_R9 (%_ASM_AX), %r9 + mov VCPU_R10(%_ASM_AX), %r10 + mov VCPU_R11(%_ASM_AX), %r11 + mov VCPU_R12(%_ASM_AX), %r12 + mov VCPU_R13(%_ASM_AX), %r13 + mov VCPU_R14(%_ASM_AX), %r14 + mov VCPU_R15(%_ASM_AX), %r15 +#endif + /* Load guest RAX. This kills the vmx_vcpu pointer! */ + mov VCPU_RAX(%_ASM_AX), %_ASM_AX + + /* Enter guest mode */ + call vmx_vmenter + + /* Jump on VM-Fail. */ + jbe 2f + + /* Temporarily save guest's RAX. */ + push %_ASM_AX + + /* Reload @regs to RAX. */ + mov WORD_SIZE(%_ASM_SP), %_ASM_AX + + /* Save all guest registers, including RAX from the stack */ + __ASM_SIZE(pop) VCPU_RAX(%_ASM_AX) + mov %_ASM_BX, VCPU_RBX(%_ASM_AX) + mov %_ASM_CX, VCPU_RCX(%_ASM_AX) + mov %_ASM_DX, VCPU_RDX(%_ASM_AX) + mov %_ASM_SI, VCPU_RSI(%_ASM_AX) + mov %_ASM_DI, VCPU_RDI(%_ASM_AX) + mov %_ASM_BP, VCPU_RBP(%_ASM_AX) +#ifdef CONFIG_X86_64 + mov %r8, VCPU_R8 (%_ASM_AX) + mov %r9, VCPU_R9 (%_ASM_AX) + mov %r10, VCPU_R10(%_ASM_AX) + mov %r11, VCPU_R11(%_ASM_AX) + mov %r12, VCPU_R12(%_ASM_AX) + mov %r13, VCPU_R13(%_ASM_AX) + mov %r14, VCPU_R14(%_ASM_AX) + mov %r15, VCPU_R15(%_ASM_AX) +#endif + + /* Clear RAX to indicate VM-Exit (as opposed to VM-Fail). */ + xor %eax, %eax + + /* + * Clear all general purpose registers except RSP and RAX to prevent + * speculative use of the guest's values, even those that are reloaded + * via the stack. In theory, an L1 cache miss when restoring registers + * could lead to speculative execution with the guest's values. + * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially + * free. RSP and RAX are exempt as RSP is restored by hardware during + * VM-Exit and RAX is explicitly loaded with 0 or 1 to return VM-Fail. + */ +1: xor %ebx, %ebx + xor %ecx, %ecx + xor %edx, %edx + xor %esi, %esi + xor %edi, %edi + xor %ebp, %ebp +#ifdef CONFIG_X86_64 + xor %r8d, %r8d + xor %r9d, %r9d + xor %r10d, %r10d + xor %r11d, %r11d + xor %r12d, %r12d + xor %r13d, %r13d + xor %r14d, %r14d + xor %r15d, %r15d +#endif + + /* "POP" @regs. */ + add $WORD_SIZE, %_ASM_SP + pop %_ASM_BX + +#ifdef CONFIG_X86_64 + pop %r12 + pop %r13 + pop %r14 + pop %r15 +#else + pop %esi + pop %edi +#endif + pop %_ASM_BP + ret + + /* VM-Fail. Out-of-line to avoid a taken Jcc after VM-Exit. */ +2: mov $1, %eax + jmp 1b +ENDPROC(__vmx_vcpu_run) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 30a6bcd735ec3..c73375e01ab8c 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -246,6 +246,10 @@ static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) { + /* + * This allocation for vmx_l1d_flush_pages is not tied to a VM + * lifetime and so should not be charged to a memcg. + */ page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER); if (!page) return -ENOMEM; @@ -2387,13 +2391,13 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, return 0; } -struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu) +struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) { int node = cpu_to_node(cpu); struct page *pages; struct vmcs *vmcs; - pages = __alloc_pages_node(node, GFP_KERNEL, vmcs_config.order); + pages = __alloc_pages_node(node, flags, vmcs_config.order); if (!pages) return NULL; vmcs = page_address(pages); @@ -2440,7 +2444,8 @@ int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) loaded_vmcs_init(loaded_vmcs); if (cpu_has_vmx_msr_bitmap()) { - loaded_vmcs->msr_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL); + loaded_vmcs->msr_bitmap = (unsigned long *) + __get_free_page(GFP_KERNEL_ACCOUNT); if (!loaded_vmcs->msr_bitmap) goto out_vmcs; memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE); @@ -2481,7 +2486,7 @@ static __init int alloc_kvm_area(void) for_each_possible_cpu(cpu) { struct vmcs *vmcs; - vmcs = alloc_vmcs_cpu(false, cpu); + vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL); if (!vmcs) { free_kvm_area(); return -ENOMEM; @@ -6360,150 +6365,15 @@ static void vmx_update_hv_timer(struct kvm_vcpu *vcpu) vmx->loaded_vmcs->hv_timer_armed = false; } -static void __vmx_vcpu_run(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx) +void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) { - unsigned long evmcs_rsp; - - vmx->__launched = vmx->loaded_vmcs->launched; - - evmcs_rsp = static_branch_unlikely(&enable_evmcs) ? - (unsigned long)¤t_evmcs->host_rsp : 0; - - if (static_branch_unlikely(&vmx_l1d_should_flush)) - vmx_l1d_flush(vcpu); - - asm( - /* Store host registers */ - "push %%" _ASM_DX "; push %%" _ASM_BP ";" - "push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */ - "push %%" _ASM_CX " \n\t" - "sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */ - "cmp %%" _ASM_SP ", %c[host_rsp](%%" _ASM_CX ") \n\t" - "je 1f \n\t" - "mov %%" _ASM_SP ", %c[host_rsp](%%" _ASM_CX ") \n\t" - /* Avoid VMWRITE when Enlightened VMCS is in use */ - "test %%" _ASM_SI ", %%" _ASM_SI " \n\t" - "jz 2f \n\t" - "mov %%" _ASM_SP ", (%%" _ASM_SI ") \n\t" - "jmp 1f \n\t" - "2: \n\t" - __ex("vmwrite %%" _ASM_SP ", %%" _ASM_DX) "\n\t" - "1: \n\t" - "add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */ - - /* Reload cr2 if changed */ - "mov %c[cr2](%%" _ASM_CX "), %%" _ASM_AX " \n\t" - "mov %%cr2, %%" _ASM_DX " \n\t" - "cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t" - "je 3f \n\t" - "mov %%" _ASM_AX", %%cr2 \n\t" - "3: \n\t" - /* Check if vmlaunch or vmresume is needed */ - "cmpl $0, %c[launched](%%" _ASM_CX ") \n\t" - /* Load guest registers. Don't clobber flags. */ - "mov %c[rax](%%" _ASM_CX "), %%" _ASM_AX " \n\t" - "mov %c[rbx](%%" _ASM_CX "), %%" _ASM_BX " \n\t" - "mov %c[rdx](%%" _ASM_CX "), %%" _ASM_DX " \n\t" - "mov %c[rsi](%%" _ASM_CX "), %%" _ASM_SI " \n\t" - "mov %c[rdi](%%" _ASM_CX "), %%" _ASM_DI " \n\t" - "mov %c[rbp](%%" _ASM_CX "), %%" _ASM_BP " \n\t" -#ifdef CONFIG_X86_64 - "mov %c[r8](%%" _ASM_CX "), %%r8 \n\t" - "mov %c[r9](%%" _ASM_CX "), %%r9 \n\t" - "mov %c[r10](%%" _ASM_CX "), %%r10 \n\t" - "mov %c[r11](%%" _ASM_CX "), %%r11 \n\t" - "mov %c[r12](%%" _ASM_CX "), %%r12 \n\t" - "mov %c[r13](%%" _ASM_CX "), %%r13 \n\t" - "mov %c[r14](%%" _ASM_CX "), %%r14 \n\t" - "mov %c[r15](%%" _ASM_CX "), %%r15 \n\t" -#endif - /* Load guest RCX. This kills the vmx_vcpu pointer! */ - "mov %c[rcx](%%" _ASM_CX "), %%" _ASM_CX " \n\t" - - /* Enter guest mode */ - "call vmx_vmenter\n\t" - - /* Save guest's RCX to the stack placeholder (see above) */ - "mov %%" _ASM_CX ", %c[wordsize](%%" _ASM_SP ") \n\t" - - /* Load host's RCX, i.e. the vmx_vcpu pointer */ - "pop %%" _ASM_CX " \n\t" - - /* Set vmx->fail based on EFLAGS.{CF,ZF} */ - "setbe %c[fail](%%" _ASM_CX ")\n\t" - - /* Save all guest registers, including RCX from the stack */ - "mov %%" _ASM_AX ", %c[rax](%%" _ASM_CX ") \n\t" - "mov %%" _ASM_BX ", %c[rbx](%%" _ASM_CX ") \n\t" - __ASM_SIZE(pop) " %c[rcx](%%" _ASM_CX ") \n\t" - "mov %%" _ASM_DX ", %c[rdx](%%" _ASM_CX ") \n\t" - "mov %%" _ASM_SI ", %c[rsi](%%" _ASM_CX ") \n\t" - "mov %%" _ASM_DI ", %c[rdi](%%" _ASM_CX ") \n\t" - "mov %%" _ASM_BP ", %c[rbp](%%" _ASM_CX ") \n\t" -#ifdef CONFIG_X86_64 - "mov %%r8, %c[r8](%%" _ASM_CX ") \n\t" - "mov %%r9, %c[r9](%%" _ASM_CX ") \n\t" - "mov %%r10, %c[r10](%%" _ASM_CX ") \n\t" - "mov %%r11, %c[r11](%%" _ASM_CX ") \n\t" - "mov %%r12, %c[r12](%%" _ASM_CX ") \n\t" - "mov %%r13, %c[r13](%%" _ASM_CX ") \n\t" - "mov %%r14, %c[r14](%%" _ASM_CX ") \n\t" - "mov %%r15, %c[r15](%%" _ASM_CX ") \n\t" - /* - * Clear host registers marked as clobbered to prevent - * speculative use. - */ - "xor %%r8d, %%r8d \n\t" - "xor %%r9d, %%r9d \n\t" - "xor %%r10d, %%r10d \n\t" - "xor %%r11d, %%r11d \n\t" - "xor %%r12d, %%r12d \n\t" - "xor %%r13d, %%r13d \n\t" - "xor %%r14d, %%r14d \n\t" - "xor %%r15d, %%r15d \n\t" -#endif - "mov %%cr2, %%" _ASM_AX " \n\t" - "mov %%" _ASM_AX ", %c[cr2](%%" _ASM_CX ") \n\t" - - "xor %%eax, %%eax \n\t" - "xor %%ebx, %%ebx \n\t" - "xor %%esi, %%esi \n\t" - "xor %%edi, %%edi \n\t" - "pop %%" _ASM_BP "; pop %%" _ASM_DX " \n\t" - : ASM_CALL_CONSTRAINT - : "c"(vmx), "d"((unsigned long)HOST_RSP), "S"(evmcs_rsp), - [launched]"i"(offsetof(struct vcpu_vmx, __launched)), - [fail]"i"(offsetof(struct vcpu_vmx, fail)), - [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)), - [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])), - [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])), -#ifdef CONFIG_X86_64 - [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])), - [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])), - [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])), -#endif - [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)), - [wordsize]"i"(sizeof(ulong)) - : "cc", "memory" -#ifdef CONFIG_X86_64 - , "rax", "rbx", "rdi" - , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" -#else - , "eax", "ebx", "edi" -#endif - ); + if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) { + vmx->loaded_vmcs->host_state.rsp = host_rsp; + vmcs_writel(HOST_RSP, host_rsp); + } } -STACK_FRAME_NON_STANDARD(__vmx_vcpu_run); + +bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs, bool launched); static void vmx_vcpu_run(struct kvm_vcpu *vcpu) { @@ -6572,7 +6442,16 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) */ x86_spec_ctrl_set_guest(vmx->spec_ctrl, 0); - __vmx_vcpu_run(vcpu, vmx); + if (static_branch_unlikely(&vmx_l1d_should_flush)) + vmx_l1d_flush(vcpu); + + if (vcpu->arch.cr2 != read_cr2()) + write_cr2(vcpu->arch.cr2); + + vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, + vmx->loaded_vmcs->launched); + + vcpu->arch.cr2 = read_cr2(); /* * We do not use IBRS in the kernel. If this vCPU has used the @@ -6657,7 +6536,9 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) static struct kvm *vmx_vm_alloc(void) { - struct kvm_vmx *kvm_vmx = vzalloc(sizeof(struct kvm_vmx)); + struct kvm_vmx *kvm_vmx = __vmalloc(sizeof(struct kvm_vmx), + GFP_KERNEL_ACCOUNT | __GFP_ZERO, + PAGE_KERNEL); return &kvm_vmx->kvm; } @@ -6673,7 +6554,6 @@ static void vmx_free_vcpu(struct kvm_vcpu *vcpu) if (enable_pml) vmx_destroy_pml_buffer(vmx); free_vpid(vmx->vpid); - leave_guest_mode(vcpu); nested_vmx_free_vcpu(vcpu); free_loaded_vmcs(vmx->loaded_vmcs); kfree(vmx->guest_msrs); @@ -6685,14 +6565,16 @@ static void vmx_free_vcpu(struct kvm_vcpu *vcpu) static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) { int err; - struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); + struct vcpu_vmx *vmx; unsigned long *msr_bitmap; int cpu; + vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT); if (!vmx) return ERR_PTR(-ENOMEM); - vmx->vcpu.arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache, GFP_KERNEL); + vmx->vcpu.arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache, + GFP_KERNEL_ACCOUNT); if (!vmx->vcpu.arch.guest_fpu) { printk(KERN_ERR "kvm: failed to allocate vcpu's fpu\n"); err = -ENOMEM; @@ -6714,12 +6596,12 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) * for the guest, etc. */ if (enable_pml) { - vmx->pml_pg = alloc_page(GFP_KERNEL | __GFP_ZERO); + vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); if (!vmx->pml_pg) goto uninit_vcpu; } - vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL); + vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); BUILD_BUG_ON(ARRAY_SIZE(vmx_msr_index) * sizeof(vmx->guest_msrs[0]) > PAGE_SIZE); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 0ac0a64c7790f..1554cb45b3931 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -175,7 +175,6 @@ struct nested_vmx { struct vcpu_vmx { struct kvm_vcpu vcpu; - unsigned long host_rsp; u8 fail; u8 msr_bitmap_mode; u32 exit_intr_info; @@ -209,7 +208,7 @@ struct vcpu_vmx { struct loaded_vmcs vmcs01; struct loaded_vmcs *loaded_vmcs; struct loaded_vmcs *loaded_cpu_state; - bool __launched; /* temporary, used in vmx_vcpu_run */ + struct msr_autoload { struct vmx_msrs guest; struct vmx_msrs host; @@ -339,8 +338,8 @@ static inline int pi_test_and_set_pir(int vector, struct pi_desc *pi_desc) static inline void pi_set_sn(struct pi_desc *pi_desc) { - return set_bit(POSTED_INTR_SN, - (unsigned long *)&pi_desc->control); + set_bit(POSTED_INTR_SN, + (unsigned long *)&pi_desc->control); } static inline void pi_set_on(struct pi_desc *pi_desc) @@ -445,7 +444,8 @@ static inline u32 vmx_vmentry_ctrl(void) { u32 vmentry_ctrl = vmcs_config.vmentry_ctrl; if (pt_mode == PT_MODE_SYSTEM) - vmentry_ctrl &= ~(VM_EXIT_PT_CONCEAL_PIP | VM_EXIT_CLEAR_IA32_RTIT_CTL); + vmentry_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP | + VM_ENTRY_LOAD_IA32_RTIT_CTL); /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ return vmentry_ctrl & ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | VM_ENTRY_LOAD_IA32_EFER); @@ -455,9 +455,10 @@ static inline u32 vmx_vmexit_ctrl(void) { u32 vmexit_ctrl = vmcs_config.vmexit_ctrl; if (pt_mode == PT_MODE_SYSTEM) - vmexit_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP | VM_ENTRY_LOAD_IA32_RTIT_CTL); + vmexit_ctrl &= ~(VM_EXIT_PT_CONCEAL_PIP | + VM_EXIT_CLEAR_IA32_RTIT_CTL); /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ - return vmcs_config.vmexit_ctrl & + return vmexit_ctrl & ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER); } @@ -478,7 +479,7 @@ static inline struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu) return &(to_vmx(vcpu)->pi_desc); } -struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu); +struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags); void free_vmcs(struct vmcs *vmcs); int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs); void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs); @@ -487,7 +488,8 @@ void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs); static inline struct vmcs *alloc_vmcs(bool shadow) { - return alloc_vmcs_cpu(shadow, raw_smp_processor_id()); + return alloc_vmcs_cpu(shadow, raw_smp_processor_id(), + GFP_KERNEL_ACCOUNT); } u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 941f932373d03..65e4559eef2fc 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3879,7 +3879,8 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = -EINVAL; if (!lapic_in_kernel(vcpu)) goto out; - u.lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL); + u.lapic = kzalloc(sizeof(struct kvm_lapic_state), + GFP_KERNEL_ACCOUNT); r = -ENOMEM; if (!u.lapic) @@ -4066,7 +4067,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, break; } case KVM_GET_XSAVE: { - u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL); + u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT); r = -ENOMEM; if (!u.xsave) break; @@ -4090,7 +4091,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, break; } case KVM_GET_XCRS: { - u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL); + u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT); r = -ENOMEM; if (!u.xcrs) break; @@ -7055,6 +7056,13 @@ static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid) void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu) { + if (!lapic_in_kernel(vcpu)) { + WARN_ON_ONCE(vcpu->arch.apicv_active); + return; + } + if (!vcpu->arch.apicv_active) + return; + vcpu->arch.apicv_active = false; kvm_x86_ops->refresh_apicv_exec_ctrl(vcpu); } @@ -9005,7 +9013,6 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) struct page *page; int r; - vcpu->arch.apicv_active = kvm_x86_ops->get_enable_apicv(vcpu); vcpu->arch.emulate_ctxt.ops = &emulate_ops; if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu)) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; @@ -9026,6 +9033,7 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) goto fail_free_pio_data; if (irqchip_in_kernel(vcpu->kvm)) { + vcpu->arch.apicv_active = kvm_x86_ops->get_enable_apicv(vcpu); r = kvm_create_lapic(vcpu); if (r < 0) goto fail_mmu_destroy; @@ -9033,14 +9041,15 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) static_key_slow_inc(&kvm_no_apic_vcpu); vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4, - GFP_KERNEL); + GFP_KERNEL_ACCOUNT); if (!vcpu->arch.mce_banks) { r = -ENOMEM; goto fail_free_lapic; } vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS; - if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL)) { + if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, + GFP_KERNEL_ACCOUNT)) { r = -ENOMEM; goto fail_free_mce_banks; } @@ -9104,7 +9113,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list); INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); - INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages); INIT_LIST_HEAD(&kvm->arch.assigned_dev_head); atomic_set(&kvm->arch.noncoherent_dma_count, 0); @@ -9299,13 +9307,13 @@ int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, slot->arch.rmap[i] = kvcalloc(lpages, sizeof(*slot->arch.rmap[i]), - GFP_KERNEL); + GFP_KERNEL_ACCOUNT); if (!slot->arch.rmap[i]) goto out_free; if (i == 0) continue; - linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL); + linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT); if (!linfo) goto out_free; @@ -9348,13 +9356,13 @@ out_free: return -ENOMEM; } -void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots) +void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) { /* * memslots->generation has been incremented. * mmio generation may have reached its maximum value. */ - kvm_mmu_invalidate_mmio_sptes(kvm, slots); + kvm_mmu_invalidate_mmio_sptes(kvm, gen); } int kvm_arch_prepare_memory_region(struct kvm *kvm, @@ -9462,7 +9470,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, void kvm_arch_flush_shadow_all(struct kvm *kvm) { - kvm_mmu_invalidate_zap_all_pages(kvm); + kvm_mmu_zap_all(kvm); } void kvm_arch_flush_shadow_memslot(struct kvm *kvm, diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 224cd0a475684..28406aa1136d7 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -181,6 +181,11 @@ static inline bool emul_is_noncanonical_address(u64 la, static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn, unsigned access) { + u64 gen = kvm_memslots(vcpu->kvm)->generation; + + if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS)) + return; + /* * If this is a shadow nested page table, the "GVA" is * actually a nGPA. @@ -188,7 +193,7 @@ static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu, vcpu->arch.mmio_gva = mmu_is_nested(vcpu) ? 0 : gva & PAGE_MASK; vcpu->arch.access = access; vcpu->arch.mmio_gfn = gfn; - vcpu->arch.mmio_gen = kvm_memslots(vcpu->kvm)->generation; + vcpu->arch.mmio_gen = gen; } static inline bool vcpu_match_mmio_gen(struct kvm_vcpu *vcpu) diff --git a/block/blk-merge.c b/block/blk-merge.c index 22467f475ab41..1c9d4f0f96eaf 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -180,7 +180,7 @@ static unsigned get_max_segment_size(struct request_queue *q, */ static bool bvec_split_segs(struct request_queue *q, struct bio_vec *bv, unsigned *nsegs, unsigned *last_seg_size, - unsigned *front_seg_size, unsigned *sectors) + unsigned *front_seg_size, unsigned *sectors, unsigned max_segs) { unsigned len = bv->bv_len; unsigned total_len = 0; @@ -190,7 +190,7 @@ static bool bvec_split_segs(struct request_queue *q, struct bio_vec *bv, * Multi-page bvec may be too big to hold in one segment, so the * current bvec has to be splitted as multiple segments. */ - while (len && new_nsegs + *nsegs < queue_max_segments(q)) { + while (len && new_nsegs + *nsegs < max_segs) { seg_size = get_max_segment_size(q, bv->bv_offset + total_len); seg_size = min(seg_size, len); @@ -240,6 +240,7 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, bool do_split = true; struct bio *new = NULL; const unsigned max_sectors = get_max_io_size(q, bio); + const unsigned max_segs = queue_max_segments(q); bio_for_each_bvec(bv, bio, iter) { /* @@ -254,14 +255,14 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, * Consider this a new segment if we're splitting in * the middle of this vector. */ - if (nsegs < queue_max_segments(q) && + if (nsegs < max_segs && sectors < max_sectors) { /* split in the middle of bvec */ bv.bv_len = (max_sectors - sectors) << 9; bvec_split_segs(q, &bv, &nsegs, &seg_size, &front_seg_size, - §ors); + §ors, max_segs); } goto split; } @@ -283,7 +284,7 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, continue; } new_segment: - if (nsegs == queue_max_segments(q)) + if (nsegs == max_segs) goto split; bvprv = bv; @@ -296,7 +297,7 @@ new_segment: if (nsegs == 1 && seg_size > front_seg_size) front_seg_size = seg_size; } else if (bvec_split_segs(q, &bv, &nsegs, &seg_size, - &front_seg_size, §ors)) { + &front_seg_size, §ors, max_segs)) { goto split; } } @@ -415,7 +416,7 @@ new_segment: bvprv = bv; prev = 1; bvec_split_segs(q, &bv, &nr_phys_segs, &seg_size, - &front_seg_size, NULL); + &front_seg_size, NULL, UINT_MAX); } bbio = bio; } diff --git a/drivers/acpi/acpi_configfs.c b/drivers/acpi/acpi_configfs.c index b588503890941..81bfc61972931 100644 --- a/drivers/acpi/acpi_configfs.c +++ b/drivers/acpi/acpi_configfs.c @@ -97,12 +97,12 @@ static ssize_t acpi_table_aml_read(struct config_item *cfg, CONFIGFS_BIN_ATTR(acpi_table_, aml, NULL, MAX_ACPI_TABLE_SIZE); -struct configfs_bin_attribute *acpi_table_bin_attrs[] = { +static struct configfs_bin_attribute *acpi_table_bin_attrs[] = { &acpi_table_attr_aml, NULL, }; -ssize_t acpi_table_signature_show(struct config_item *cfg, char *str) +static ssize_t acpi_table_signature_show(struct config_item *cfg, char *str) { struct acpi_table_header *h = get_header(cfg); @@ -112,7 +112,7 @@ ssize_t acpi_table_signature_show(struct config_item *cfg, char *str) return sprintf(str, "%.*s\n", ACPI_NAME_SIZE, h->signature); } -ssize_t acpi_table_length_show(struct config_item *cfg, char *str) +static ssize_t acpi_table_length_show(struct config_item *cfg, char *str) { struct acpi_table_header *h = get_header(cfg); @@ -122,7 +122,7 @@ ssize_t acpi_table_length_show(struct config_item *cfg, char *str) return sprintf(str, "%d\n", h->length); } -ssize_t acpi_table_revision_show(struct config_item *cfg, char *str) +static ssize_t acpi_table_revision_show(struct config_item *cfg, char *str) { struct acpi_table_header *h = get_header(cfg); @@ -132,7 +132,7 @@ ssize_t acpi_table_revision_show(struct config_item *cfg, char *str) return sprintf(str, "%d\n", h->revision); } -ssize_t acpi_table_oem_id_show(struct config_item *cfg, char *str) +static ssize_t acpi_table_oem_id_show(struct config_item *cfg, char *str) { struct acpi_table_header *h = get_header(cfg); @@ -142,7 +142,7 @@ ssize_t acpi_table_oem_id_show(struct config_item *cfg, char *str) return sprintf(str, "%.*s\n", ACPI_OEM_ID_SIZE, h->oem_id); } -ssize_t acpi_table_oem_table_id_show(struct config_item *cfg, char *str) +static ssize_t acpi_table_oem_table_id_show(struct config_item *cfg, char *str) { struct acpi_table_header *h = get_header(cfg); @@ -152,7 +152,7 @@ ssize_t acpi_table_oem_table_id_show(struct config_item *cfg, char *str) return sprintf(str, "%.*s\n", ACPI_OEM_TABLE_ID_SIZE, h->oem_table_id); } -ssize_t acpi_table_oem_revision_show(struct config_item *cfg, char *str) +static ssize_t acpi_table_oem_revision_show(struct config_item *cfg, char *str) { struct acpi_table_header *h = get_header(cfg); @@ -162,7 +162,8 @@ ssize_t acpi_table_oem_revision_show(struct config_item *cfg, char *str) return sprintf(str, "%d\n", h->oem_revision); } -ssize_t acpi_table_asl_compiler_id_show(struct config_item *cfg, char *str) +static ssize_t acpi_table_asl_compiler_id_show(struct config_item *cfg, + char *str) { struct acpi_table_header *h = get_header(cfg); @@ -172,8 +173,8 @@ ssize_t acpi_table_asl_compiler_id_show(struct config_item *cfg, char *str) return sprintf(str, "%.*s\n", ACPI_NAME_SIZE, h->asl_compiler_id); } -ssize_t acpi_table_asl_compiler_revision_show(struct config_item *cfg, - char *str) +static ssize_t acpi_table_asl_compiler_revision_show(struct config_item *cfg, + char *str) { struct acpi_table_header *h = get_header(cfg); @@ -192,7 +193,7 @@ CONFIGFS_ATTR_RO(acpi_table_, oem_revision); CONFIGFS_ATTR_RO(acpi_table_, asl_compiler_id); CONFIGFS_ATTR_RO(acpi_table_, asl_compiler_revision); -struct configfs_attribute *acpi_table_attrs[] = { +static struct configfs_attribute *acpi_table_attrs[] = { &acpi_table_attr_signature, &acpi_table_attr_length, &acpi_table_attr_revision, @@ -232,7 +233,7 @@ static void acpi_table_drop_item(struct config_group *group, acpi_tb_unload_table(table->index); } -struct configfs_group_operations acpi_table_group_ops = { +static struct configfs_group_operations acpi_table_group_ops = { .make_item = acpi_table_make_item, .drop_item = acpi_table_drop_item, }; diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c index 5f94c35d165fe..1e2a10a06b9dc 100644 --- a/drivers/acpi/acpi_lpss.c +++ b/drivers/acpi/acpi_lpss.c @@ -18,7 +18,7 @@ #include <linux/mutex.h> #include <linux/pci.h> #include <linux/platform_device.h> -#include <linux/platform_data/clk-lpss.h> +#include <linux/platform_data/x86/clk-lpss.h> #include <linux/platform_data/x86/pmc_atom.h> #include <linux/pm_domain.h> #include <linux/pm_runtime.h> diff --git a/drivers/acpi/device_sysfs.c b/drivers/acpi/device_sysfs.c index 545e91420cded..8940054d6250f 100644 --- a/drivers/acpi/device_sysfs.c +++ b/drivers/acpi/device_sysfs.c @@ -202,11 +202,15 @@ static int create_of_modalias(struct acpi_device *acpi_dev, char *modalias, { struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER }; const union acpi_object *of_compatible, *obj; + acpi_status status; int len, count; int i, nval; char *c; - acpi_get_name(acpi_dev->handle, ACPI_SINGLE_NAME, &buf); + status = acpi_get_name(acpi_dev->handle, ACPI_SINGLE_NAME, &buf); + if (ACPI_FAILURE(status)) + return -ENODEV; + /* DT strings are all in lower case */ for (c = buf.pointer; *c != '\0'; c++) *c = tolower(*c); diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index df8979008dd4e..5a389a4f4f652 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -2956,11 +2956,15 @@ static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, ndr_desc->res = &res; ndr_desc->provider_data = nfit_spa; ndr_desc->attr_groups = acpi_nfit_region_attribute_groups; - if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) + if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) { ndr_desc->numa_node = acpi_map_pxm_to_online_node( spa->proximity_domain); - else + ndr_desc->target_node = acpi_map_pxm_to_node( + spa->proximity_domain); + } else { ndr_desc->numa_node = NUMA_NO_NODE; + ndr_desc->target_node = NUMA_NO_NODE; + } /* * Persistence domain bits are hierarchical, if diff --git a/drivers/acpi/numa.c b/drivers/acpi/numa.c index 7bbbf8256a41a..867f6e3f2b4f4 100644 --- a/drivers/acpi/numa.c +++ b/drivers/acpi/numa.c @@ -84,6 +84,7 @@ int acpi_map_pxm_to_node(int pxm) return node; } +EXPORT_SYMBOL(acpi_map_pxm_to_node); /** * acpi_map_pxm_to_online_node - Map proximity ID to online node diff --git a/drivers/acpi/pptt.c b/drivers/acpi/pptt.c index ad31c50de3be8..065c4fc245d11 100644 --- a/drivers/acpi/pptt.c +++ b/drivers/acpi/pptt.c @@ -209,6 +209,9 @@ static int acpi_pptt_leaf_node(struct acpi_table_header *table_hdr, struct acpi_pptt_processor *cpu_node; u32 proc_sz; + if (table_hdr->revision > 1) + return (node->flags & ACPI_PPTT_ACPI_LEAF_NODE); + table_end = (unsigned long)table_hdr + table_hdr->length; node_entry = ACPI_PTR_DIFF(node, table_hdr); entry = ACPI_ADD_PTR(struct acpi_subtable_header, table_hdr, diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c index 41324f0b1bee2..fa76f5e41b5ca 100644 --- a/drivers/acpi/sysfs.c +++ b/drivers/acpi/sysfs.c @@ -648,26 +648,29 @@ static void acpi_global_event_handler(u32 event_type, acpi_handle device, } } -static int get_status(u32 index, acpi_event_status *status, +static int get_status(u32 index, acpi_event_status *ret, acpi_handle *handle) { - int result; + acpi_status status; if (index >= num_gpes + ACPI_NUM_FIXED_EVENTS) return -EINVAL; if (index < num_gpes) { - result = acpi_get_gpe_device(index, handle); - if (result) { + status = acpi_get_gpe_device(index, handle); + if (ACPI_FAILURE(status)) { ACPI_EXCEPTION((AE_INFO, AE_NOT_FOUND, "Invalid GPE 0x%x", index)); - return result; + return -ENXIO; } - result = acpi_get_gpe_status(*handle, index, status); - } else if (index < (num_gpes + ACPI_NUM_FIXED_EVENTS)) - result = acpi_get_event_status(index - num_gpes, status); + status = acpi_get_gpe_status(*handle, index, ret); + } else { + status = acpi_get_event_status(index - num_gpes, ret); + } + if (ACPI_FAILURE(status)) + return -EIO; - return result; + return 0; } static ssize_t counter_show(struct kobject *kobj, diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index 41b706403ef72..b4dae624b9afe 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -26,19 +26,36 @@ #define to_amba_driver(d) container_of(d, struct amba_driver, drv) -static const struct amba_id * -amba_lookup(const struct amba_id *table, struct amba_device *dev) +/* called on periphid match and class 0x9 coresight device. */ +static int +amba_cs_uci_id_match(const struct amba_id *table, struct amba_device *dev) { int ret = 0; + struct amba_cs_uci_id *uci; + + uci = table->data; + /* no table data or zero mask - return match on periphid */ + if (!uci || (uci->devarch_mask == 0)) + return 1; + + /* test against read devtype and masked devarch value */ + ret = (dev->uci.devtype == uci->devtype) && + ((dev->uci.devarch & uci->devarch_mask) == uci->devarch); + return ret; +} + +static const struct amba_id * +amba_lookup(const struct amba_id *table, struct amba_device *dev) +{ while (table->mask) { - ret = (dev->periphid & table->mask) == table->id; - if (ret) - break; + if (((dev->periphid & table->mask) == table->id) && + ((dev->cid != CORESIGHT_CID) || + (amba_cs_uci_id_match(table, dev)))) + return table; table++; } - - return ret ? table : NULL; + return NULL; } static int amba_match(struct device *dev, struct device_driver *drv) @@ -399,10 +416,22 @@ static int amba_device_try_add(struct amba_device *dev, struct resource *parent) cid |= (readl(tmp + size - 0x10 + 4 * i) & 255) << (i * 8); + if (cid == CORESIGHT_CID) { + /* set the base to the start of the last 4k block */ + void __iomem *csbase = tmp + size - 4096; + + dev->uci.devarch = + readl(csbase + UCI_REG_DEVARCH_OFFSET); + dev->uci.devtype = + readl(csbase + UCI_REG_DEVTYPE_OFFSET) & 0xff; + } + amba_put_disable_pclk(dev); - if (cid == AMBA_CID || cid == CORESIGHT_CID) + if (cid == AMBA_CID || cid == CORESIGHT_CID) { dev->periphid = pid; + dev->cid = cid; + } if (!dev->periphid) ret = -ENODEV; diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 048cbf7d5233b..cb8347500ce28 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -88,6 +88,7 @@ unsigned long __weak memory_block_size_bytes(void) { return MIN_MEMORY_BLOCK_SIZE; } +EXPORT_SYMBOL_GPL(memory_block_size_bytes); static unsigned long get_memory_block_size(void) { diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 2c334c01fc43c..76c9969b7124c 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -6,6 +6,8 @@ * This file is released under the GPLv2. */ +#define pr_fmt(fmt) "PM: " fmt + #include <linux/delay.h> #include <linux/kernel.h> #include <linux/io.h> @@ -457,19 +459,19 @@ static int _genpd_power_off(struct generic_pm_domain *genpd, bool timed) time_start = ktime_get(); ret = genpd->power_off(genpd); - if (ret == -EBUSY) + if (ret) return ret; elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start)); if (elapsed_ns <= genpd->states[state_idx].power_off_latency_ns) - return ret; + return 0; genpd->states[state_idx].power_off_latency_ns = elapsed_ns; genpd->max_off_time_changed = true; pr_debug("%s: Power-%s latency exceeded, new value %lld ns\n", genpd->name, "off", elapsed_ns); - return ret; + return 0; } /** @@ -1657,8 +1659,8 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd, genpd_lock_nested(genpd, SINGLE_DEPTH_NESTING); if (!list_empty(&subdomain->master_links) || subdomain->device_count) { - pr_warn("%s: unable to remove subdomain %s\n", genpd->name, - subdomain->name); + pr_warn("%s: unable to remove subdomain %s\n", + genpd->name, subdomain->name); ret = -EBUSY; goto out; } @@ -1766,8 +1768,8 @@ int pm_genpd_init(struct generic_pm_domain *genpd, ret = genpd_set_default_power_state(genpd); if (ret) return ret; - } else if (!gov) { - pr_warn("%s : no governor for states\n", genpd->name); + } else if (!gov && genpd->state_count > 1) { + pr_warn("%s: no governor for states\n", genpd->name); } device_initialize(&genpd->dev); @@ -2514,7 +2516,7 @@ static int genpd_parse_state(struct genpd_power_state *genpd_state, &entry_latency); if (err) { pr_debug(" * %pOF missing entry-latency-us property\n", - state_node); + state_node); return -EINVAL; } @@ -2522,7 +2524,7 @@ static int genpd_parse_state(struct genpd_power_state *genpd_state, &exit_latency); if (err) { pr_debug(" * %pOF missing exit-latency-us property\n", - state_node); + state_node); return -EINVAL; } diff --git a/drivers/base/power/domain_governor.c b/drivers/base/power/domain_governor.c index 99896fbf18e43..4d07e38a8247f 100644 --- a/drivers/base/power/domain_governor.c +++ b/drivers/base/power/domain_governor.c @@ -128,7 +128,6 @@ static bool __default_power_down_ok(struct dev_pm_domain *pd, off_on_time_ns = genpd->states[state].power_off_latency_ns + genpd->states[state].power_on_latency_ns; - min_off_time_ns = -1; /* * Check if subdomains can be off for enough time. diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 5a8149829ab39..f80d298de3fa4 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -17,6 +17,8 @@ * subsystem list maintains. */ +#define pr_fmt(fmt) "PM: " fmt + #include <linux/device.h> #include <linux/export.h> #include <linux/mutex.h> @@ -128,7 +130,7 @@ void device_pm_add(struct device *dev) if (device_pm_not_required(dev)) return; - pr_debug("PM: Adding info for %s:%s\n", + pr_debug("Adding info for %s:%s\n", dev->bus ? dev->bus->name : "No Bus", dev_name(dev)); device_pm_check_callbacks(dev); mutex_lock(&dpm_list_mtx); @@ -149,7 +151,7 @@ void device_pm_remove(struct device *dev) if (device_pm_not_required(dev)) return; - pr_debug("PM: Removing info for %s:%s\n", + pr_debug("Removing info for %s:%s\n", dev->bus ? dev->bus->name : "No Bus", dev_name(dev)); complete_all(&dev->power.completion); mutex_lock(&dpm_list_mtx); @@ -168,7 +170,7 @@ void device_pm_remove(struct device *dev) */ void device_pm_move_before(struct device *deva, struct device *devb) { - pr_debug("PM: Moving %s:%s before %s:%s\n", + pr_debug("Moving %s:%s before %s:%s\n", deva->bus ? deva->bus->name : "No Bus", dev_name(deva), devb->bus ? devb->bus->name : "No Bus", dev_name(devb)); /* Delete deva from dpm_list and reinsert before devb. */ @@ -182,7 +184,7 @@ void device_pm_move_before(struct device *deva, struct device *devb) */ void device_pm_move_after(struct device *deva, struct device *devb) { - pr_debug("PM: Moving %s:%s after %s:%s\n", + pr_debug("Moving %s:%s after %s:%s\n", deva->bus ? deva->bus->name : "No Bus", dev_name(deva), devb->bus ? devb->bus->name : "No Bus", dev_name(devb)); /* Delete deva from dpm_list and reinsert after devb. */ @@ -195,7 +197,7 @@ void device_pm_move_after(struct device *deva, struct device *devb) */ void device_pm_move_last(struct device *dev) { - pr_debug("PM: Moving %s:%s to end of list\n", + pr_debug("Moving %s:%s to end of list\n", dev->bus ? dev->bus->name : "No Bus", dev_name(dev)); list_move_tail(&dev->power.entry, &dpm_list); } @@ -418,8 +420,8 @@ static void pm_dev_dbg(struct device *dev, pm_message_t state, const char *info) static void pm_dev_err(struct device *dev, pm_message_t state, const char *info, int error) { - printk(KERN_ERR "PM: Device %s failed to %s%s: error %d\n", - dev_name(dev), pm_verb(state.event), info, error); + pr_err("Device %s failed to %s%s: error %d\n", + dev_name(dev), pm_verb(state.event), info, error); } static void dpm_show_time(ktime_t starttime, pm_message_t state, int error, @@ -2022,8 +2024,7 @@ int dpm_prepare(pm_message_t state) error = 0; continue; } - printk(KERN_INFO "PM: Device %s not prepared " - "for power transition: code %d\n", + pr_info("Device %s not prepared for power transition: code %d\n", dev_name(dev), error); put_device(dev); break; @@ -2062,7 +2063,7 @@ EXPORT_SYMBOL_GPL(dpm_suspend_start); void __suspend_report_result(const char *function, void *fn, int ret) { if (ret) - printk(KERN_ERR "%s(): %pF returns %d\n", function, fn, ret); + pr_err("%s(): %pF returns %d\n", function, fn, ret); } EXPORT_SYMBOL_GPL(__suspend_report_result); diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h index c511def48b486..ec33fbdb919b3 100644 --- a/drivers/base/power/power.h +++ b/drivers/base/power/power.h @@ -21,6 +21,7 @@ static inline void pm_runtime_early_init(struct device *dev) extern void pm_runtime_init(struct device *dev); extern void pm_runtime_reinit(struct device *dev); extern void pm_runtime_remove(struct device *dev); +extern u64 pm_runtime_active_time(struct device *dev); #define WAKE_IRQ_DEDICATED_ALLOCATED BIT(0) #define WAKE_IRQ_DEDICATED_MANAGED BIT(1) diff --git a/drivers/base/power/qos.c b/drivers/base/power/qos.c index 3382542b39b77..f80e402ef7786 100644 --- a/drivers/base/power/qos.c +++ b/drivers/base/power/qos.c @@ -22,7 +22,7 @@ * per-device constraint data struct. * * Note about the per-device constraint data struct allocation: - * . The per-device constraints data struct ptr is tored into the device + * . The per-device constraints data struct ptr is stored into the device * dev_pm_info. * . To minimize the data usage by the per-device constraints, the data struct * is only allocated at the first call to dev_pm_qos_add_request. diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index a80dbf08a99c5..977db40378b07 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -64,7 +64,7 @@ static int rpm_suspend(struct device *dev, int rpmflags); * runtime_status field is updated, to account the time in the old state * correctly. */ -void update_pm_runtime_accounting(struct device *dev) +static void update_pm_runtime_accounting(struct device *dev) { u64 now, last, delta; @@ -98,7 +98,7 @@ static void __update_runtime_status(struct device *dev, enum rpm_status status) dev->power.runtime_status = status; } -u64 pm_runtime_suspended_time(struct device *dev) +static u64 rpm_get_accounted_time(struct device *dev, bool suspended) { u64 time; unsigned long flags; @@ -106,12 +106,22 @@ u64 pm_runtime_suspended_time(struct device *dev) spin_lock_irqsave(&dev->power.lock, flags); update_pm_runtime_accounting(dev); - time = dev->power.suspended_time; + time = suspended ? dev->power.suspended_time : dev->power.active_time; spin_unlock_irqrestore(&dev->power.lock, flags); return time; } + +u64 pm_runtime_active_time(struct device *dev) +{ + return rpm_get_accounted_time(dev, false); +} + +u64 pm_runtime_suspended_time(struct device *dev) +{ + return rpm_get_accounted_time(dev, true); +} EXPORT_SYMBOL_GPL(pm_runtime_suspended_time); /** diff --git a/drivers/base/power/sysfs.c b/drivers/base/power/sysfs.c index c6bf76124184c..1226e441ddfed 100644 --- a/drivers/base/power/sysfs.c +++ b/drivers/base/power/sysfs.c @@ -125,13 +125,9 @@ static ssize_t runtime_active_time_show(struct device *dev, struct device_attribute *attr, char *buf) { int ret; - u64 tmp; - spin_lock_irq(&dev->power.lock); - update_pm_runtime_accounting(dev); - tmp = dev->power.active_time; + u64 tmp = pm_runtime_active_time(dev); do_div(tmp, NSEC_PER_MSEC); ret = sprintf(buf, "%llu\n", tmp); - spin_unlock_irq(&dev->power.lock); return ret; } @@ -141,13 +137,9 @@ static ssize_t runtime_suspended_time_show(struct device *dev, struct device_attribute *attr, char *buf) { int ret; - u64 tmp; - spin_lock_irq(&dev->power.lock); - update_pm_runtime_accounting(dev); - tmp = dev->power.suspended_time; + u64 tmp = pm_runtime_suspended_time(dev); do_div(tmp, NSEC_PER_MSEC); ret = sprintf(buf, "%llu\n", tmp); - spin_unlock_irq(&dev->power.lock); return ret; } diff --git a/drivers/base/power/trace.c b/drivers/base/power/trace.c index b11f47a1e8193..2bd9d2c744cad 100644 --- a/drivers/base/power/trace.c +++ b/drivers/base/power/trace.c @@ -7,6 +7,8 @@ * devices may be working. */ +#define pr_fmt(fmt) "PM: " fmt + #include <linux/pm-trace.h> #include <linux/export.h> #include <linux/rtc.h> diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index f1fee72ed970a..bb1ae175fae13 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -6,6 +6,8 @@ * This file is released under the GPLv2. */ +#define pr_fmt(fmt) "PM: " fmt + #include <linux/device.h> #include <linux/slab.h> #include <linux/sched/signal.h> @@ -106,23 +108,6 @@ struct wakeup_source *wakeup_source_create(const char *name) } EXPORT_SYMBOL_GPL(wakeup_source_create); -/** - * wakeup_source_drop - Prepare a struct wakeup_source object for destruction. - * @ws: Wakeup source to prepare for destruction. - * - * Callers must ensure that __pm_stay_awake() or __pm_wakeup_event() will never - * be run in parallel with this function for the same wakeup source object. - */ -void wakeup_source_drop(struct wakeup_source *ws) -{ - if (!ws) - return; - - del_timer_sync(&ws->timer); - __pm_relax(ws); -} -EXPORT_SYMBOL_GPL(wakeup_source_drop); - /* * Record wakeup_source statistics being deleted into a dummy wakeup_source. */ @@ -162,7 +147,7 @@ void wakeup_source_destroy(struct wakeup_source *ws) if (!ws) return; - wakeup_source_drop(ws); + __pm_relax(ws); wakeup_source_record(ws); kfree_const(ws->name); kfree(ws); @@ -205,6 +190,13 @@ void wakeup_source_remove(struct wakeup_source *ws) list_del_rcu(&ws->entry); raw_spin_unlock_irqrestore(&events_lock, flags); synchronize_srcu(&wakeup_srcu); + + del_timer_sync(&ws->timer); + /* + * Clear timer.function to make wakeup_source_not_registered() treat + * this wakeup source as not registered. + */ + ws->timer.function = NULL; } EXPORT_SYMBOL_GPL(wakeup_source_remove); @@ -853,7 +845,7 @@ bool pm_wakeup_pending(void) raw_spin_unlock_irqrestore(&events_lock, flags); if (ret) { - pr_debug("PM: Wakeup pending, aborting suspend\n"); + pr_debug("Wakeup pending, aborting suspend\n"); pm_print_active_wakeup_sources(); } diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c index a4bc74e72c394..24896ffb04ed8 100644 --- a/drivers/block/xen-blkback/xenbus.c +++ b/drivers/block/xen-blkback/xenbus.c @@ -926,7 +926,7 @@ static int read_per_ring_refs(struct xen_blkif_ring *ring, const char *dir) int err, i, j; struct xen_blkif *blkif = ring->blkif; struct xenbus_device *dev = blkif->be->dev; - unsigned int ring_page_order, nr_grefs, evtchn; + unsigned int nr_grefs, evtchn; err = xenbus_scanf(XBT_NIL, dir, "event-channel", "%u", &evtchn); @@ -936,43 +936,42 @@ static int read_per_ring_refs(struct xen_blkif_ring *ring, const char *dir) return err; } - err = xenbus_scanf(XBT_NIL, dev->otherend, "ring-page-order", "%u", - &ring_page_order); - if (err != 1) { - err = xenbus_scanf(XBT_NIL, dir, "ring-ref", "%u", &ring_ref[0]); + nr_grefs = blkif->nr_ring_pages; + + if (unlikely(!nr_grefs)) { + WARN_ON(true); + return -EINVAL; + } + + for (i = 0; i < nr_grefs; i++) { + char ring_ref_name[RINGREF_NAME_LEN]; + + snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i); + err = xenbus_scanf(XBT_NIL, dir, ring_ref_name, + "%u", &ring_ref[i]); + if (err != 1) { + if (nr_grefs == 1) + break; + err = -EINVAL; - xenbus_dev_fatal(dev, err, "reading %s/ring-ref", dir); + xenbus_dev_fatal(dev, err, "reading %s/%s", + dir, ring_ref_name); return err; } - nr_grefs = 1; - } else { - unsigned int i; + } - if (ring_page_order > xen_blkif_max_ring_order) { + if (err != 1) { + WARN_ON(nr_grefs != 1); + + err = xenbus_scanf(XBT_NIL, dir, "ring-ref", "%u", + &ring_ref[0]); + if (err != 1) { err = -EINVAL; - xenbus_dev_fatal(dev, err, "%s/request %d ring page order exceed max:%d", - dir, ring_page_order, - xen_blkif_max_ring_order); + xenbus_dev_fatal(dev, err, "reading %s/ring-ref", dir); return err; } - - nr_grefs = 1 << ring_page_order; - for (i = 0; i < nr_grefs; i++) { - char ring_ref_name[RINGREF_NAME_LEN]; - - snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i); - err = xenbus_scanf(XBT_NIL, dir, ring_ref_name, - "%u", &ring_ref[i]); - if (err != 1) { - err = -EINVAL; - xenbus_dev_fatal(dev, err, "reading %s/%s", - dir, ring_ref_name); - return err; - } - } } - blkif->nr_ring_pages = nr_grefs; for (i = 0; i < nr_grefs * XEN_BLKIF_REQS_PER_PAGE; i++) { req = kzalloc(sizeof(*req), GFP_KERNEL); @@ -1023,6 +1022,7 @@ fail: static int connect_ring(struct backend_info *be) { struct xenbus_device *dev = be->dev; + struct xen_blkif *blkif = be->blkif; unsigned int pers_grants; char protocol[64] = ""; int err, i; @@ -1030,28 +1030,29 @@ static int connect_ring(struct backend_info *be) size_t xspathsize; const size_t xenstore_path_ext_size = 11; /* sufficient for "/queue-NNN" */ unsigned int requested_num_queues = 0; + unsigned int ring_page_order; pr_debug("%s %s\n", __func__, dev->otherend); - be->blkif->blk_protocol = BLKIF_PROTOCOL_DEFAULT; + blkif->blk_protocol = BLKIF_PROTOCOL_DEFAULT; err = xenbus_scanf(XBT_NIL, dev->otherend, "protocol", "%63s", protocol); if (err <= 0) strcpy(protocol, "unspecified, assuming default"); else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_NATIVE)) - be->blkif->blk_protocol = BLKIF_PROTOCOL_NATIVE; + blkif->blk_protocol = BLKIF_PROTOCOL_NATIVE; else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_X86_32)) - be->blkif->blk_protocol = BLKIF_PROTOCOL_X86_32; + blkif->blk_protocol = BLKIF_PROTOCOL_X86_32; else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_X86_64)) - be->blkif->blk_protocol = BLKIF_PROTOCOL_X86_64; + blkif->blk_protocol = BLKIF_PROTOCOL_X86_64; else { xenbus_dev_fatal(dev, err, "unknown fe protocol %s", protocol); return -ENOSYS; } pers_grants = xenbus_read_unsigned(dev->otherend, "feature-persistent", 0); - be->blkif->vbd.feature_gnt_persistent = pers_grants; - be->blkif->vbd.overflow_max_grants = 0; + blkif->vbd.feature_gnt_persistent = pers_grants; + blkif->vbd.overflow_max_grants = 0; /* * Read the number of hardware queues from frontend. @@ -1067,16 +1068,30 @@ static int connect_ring(struct backend_info *be) requested_num_queues, xenblk_max_queues); return -ENOSYS; } - be->blkif->nr_rings = requested_num_queues; - if (xen_blkif_alloc_rings(be->blkif)) + blkif->nr_rings = requested_num_queues; + if (xen_blkif_alloc_rings(blkif)) return -ENOMEM; pr_info("%s: using %d queues, protocol %d (%s) %s\n", dev->nodename, - be->blkif->nr_rings, be->blkif->blk_protocol, protocol, + blkif->nr_rings, blkif->blk_protocol, protocol, pers_grants ? "persistent grants" : ""); - if (be->blkif->nr_rings == 1) - return read_per_ring_refs(&be->blkif->rings[0], dev->otherend); + ring_page_order = xenbus_read_unsigned(dev->otherend, + "ring-page-order", 0); + + if (ring_page_order > xen_blkif_max_ring_order) { + err = -EINVAL; + xenbus_dev_fatal(dev, err, + "requested ring page order %d exceed max:%d", + ring_page_order, + xen_blkif_max_ring_order); + return err; + } + + blkif->nr_ring_pages = 1 << ring_page_order; + + if (blkif->nr_rings == 1) + return read_per_ring_refs(&blkif->rings[0], dev->otherend); else { xspathsize = strlen(dev->otherend) + xenstore_path_ext_size; xspath = kmalloc(xspathsize, GFP_KERNEL); @@ -1085,10 +1100,10 @@ static int connect_ring(struct backend_info *be) return -ENOMEM; } - for (i = 0; i < be->blkif->nr_rings; i++) { + for (i = 0; i < blkif->nr_rings; i++) { memset(xspath, 0, xspathsize); snprintf(xspath, xspathsize, "%s/queue-%u", dev->otherend, i); - err = read_per_ring_refs(&be->blkif->rings[i], xspath); + err = read_per_ring_refs(&blkif->rings[i], xspath); if (err) { kfree(xspath); return err; diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 04ca65912638d..e7a5f1d1c3141 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -41,7 +41,7 @@ static DEFINE_IDR(zram_index_idr); static DEFINE_MUTEX(zram_index_mutex); static int zram_major; -static const char *default_compressor = "lzo"; +static const char *default_compressor = "lzo-rle"; /* Module params (documentation at end) */ static unsigned int num_devices = 1; diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index d2f0bb5ba47ea..e705aab9e38ba 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -290,6 +290,12 @@ config COMMON_CLK_BD718XX This driver supports ROHM BD71837 and ROHM BD71847 PMICs clock gates. +config COMMON_CLK_FIXED_MMIO + bool "Clock driver for Memory Mapped Fixed values" + depends on COMMON_CLK && OF + help + Support for Memory Mapped IO Fixed clocks + source "drivers/clk/actions/Kconfig" source "drivers/clk/bcm/Kconfig" source "drivers/clk/hisilicon/Kconfig" diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index 8a9440a975004..1db133652f0c3 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -27,6 +27,7 @@ obj-$(CONFIG_COMMON_CLK_CDCE925) += clk-cdce925.o obj-$(CONFIG_ARCH_CLPS711X) += clk-clps711x.o obj-$(CONFIG_COMMON_CLK_CS2000_CP) += clk-cs2000-cp.o obj-$(CONFIG_ARCH_EFM32) += clk-efm32gg.o +obj-$(CONFIG_COMMON_CLK_FIXED_MMIO) += clk-fixed-mmio.o obj-$(CONFIG_COMMON_CLK_GEMINI) += clk-gemini.o obj-$(CONFIG_COMMON_CLK_ASPEED) += clk-aspeed.o obj-$(CONFIG_ARCH_HIGHBANK) += clk-highbank.o @@ -78,7 +79,7 @@ obj-$(CONFIG_ARCH_K3) += keystone/ obj-$(CONFIG_ARCH_KEYSTONE) += keystone/ obj-$(CONFIG_MACH_LOONGSON32) += loongson1/ obj-y += mediatek/ -obj-$(CONFIG_COMMON_CLK_AMLOGIC) += meson/ +obj-$(CONFIG_ARCH_MESON) += meson/ obj-$(CONFIG_MACH_PIC32) += microchip/ ifeq ($(CONFIG_COMMON_CLK), y) obj-$(CONFIG_ARCH_MMP) += mmp/ diff --git a/drivers/clk/actions/Kconfig b/drivers/clk/actions/Kconfig index 04f0a6355726f..5b45ca35757e8 100644 --- a/drivers/clk/actions/Kconfig +++ b/drivers/clk/actions/Kconfig @@ -9,6 +9,11 @@ if CLK_ACTIONS # SoC Drivers +config CLK_OWL_S500 + bool "Support for the Actions Semi OWL S500 clocks" + depends on ARCH_ACTIONS || COMPILE_TEST + default ARCH_ACTIONS + config CLK_OWL_S700 bool "Support for the Actions Semi OWL S700 clocks" depends on (ARM64 && ARCH_ACTIONS) || COMPILE_TEST diff --git a/drivers/clk/actions/Makefile b/drivers/clk/actions/Makefile index ccfdf9781ceff..a2588e55c7902 100644 --- a/drivers/clk/actions/Makefile +++ b/drivers/clk/actions/Makefile @@ -10,5 +10,6 @@ clk-owl-y += owl-pll.o clk-owl-y += owl-reset.o # SoC support +obj-$(CONFIG_CLK_OWL_S500) += owl-s500.o obj-$(CONFIG_CLK_OWL_S700) += owl-s700.o obj-$(CONFIG_CLK_OWL_S900) += owl-s900.o diff --git a/drivers/clk/actions/owl-pll.c b/drivers/clk/actions/owl-pll.c index 058e06d7099f6..02437bdedf4db 100644 --- a/drivers/clk/actions/owl-pll.c +++ b/drivers/clk/actions/owl-pll.c @@ -179,7 +179,7 @@ static int owl_pll_set_rate(struct clk_hw *hw, unsigned long rate, regmap_write(common->regmap, pll_hw->reg, reg); - udelay(PLL_STABILITY_WAIT_US); + udelay(pll_hw->delay); return 0; } diff --git a/drivers/clk/actions/owl-pll.h b/drivers/clk/actions/owl-pll.h index 0aae30abd5dc7..6fb0d45bb0882 100644 --- a/drivers/clk/actions/owl-pll.h +++ b/drivers/clk/actions/owl-pll.h @@ -13,6 +13,8 @@ #include "owl-common.h" +#define OWL_PLL_DEF_DELAY 50 + /* last entry should have rate = 0 */ struct clk_pll_table { unsigned int val; @@ -27,6 +29,7 @@ struct owl_pll_hw { u8 width; u8 min_mul; u8 max_mul; + u8 delay; const struct clk_pll_table *table; }; @@ -36,7 +39,7 @@ struct owl_pll { }; #define OWL_PLL_HW(_reg, _bfreq, _bit_idx, _shift, \ - _width, _min_mul, _max_mul, _table) \ + _width, _min_mul, _max_mul, _delay, _table) \ { \ .reg = _reg, \ .bfreq = _bfreq, \ @@ -45,6 +48,7 @@ struct owl_pll { .width = _width, \ .min_mul = _min_mul, \ .max_mul = _max_mul, \ + .delay = _delay, \ .table = _table, \ } @@ -52,8 +56,8 @@ struct owl_pll { _shift, _width, _min_mul, _max_mul, _table, _flags) \ struct owl_pll _struct = { \ .pll_hw = OWL_PLL_HW(_reg, _bfreq, _bit_idx, _shift, \ - _width, _min_mul, \ - _max_mul, _table), \ + _width, _min_mul, _max_mul, \ + OWL_PLL_DEF_DELAY, _table), \ .common = { \ .regmap = NULL, \ .hw.init = CLK_HW_INIT(_name, \ @@ -67,8 +71,23 @@ struct owl_pll { _shift, _width, _min_mul, _max_mul, _table, _flags) \ struct owl_pll _struct = { \ .pll_hw = OWL_PLL_HW(_reg, _bfreq, _bit_idx, _shift, \ - _width, _min_mul, \ - _max_mul, _table), \ + _width, _min_mul, _max_mul, \ + OWL_PLL_DEF_DELAY, _table), \ + .common = { \ + .regmap = NULL, \ + .hw.init = CLK_HW_INIT_NO_PARENT(_name, \ + &owl_pll_ops, \ + _flags), \ + }, \ + } + +#define OWL_PLL_NO_PARENT_DELAY(_struct, _name, _reg, _bfreq, _bit_idx, \ + _shift, _width, _min_mul, _max_mul, _delay, _table, \ + _flags) \ + struct owl_pll _struct = { \ + .pll_hw = OWL_PLL_HW(_reg, _bfreq, _bit_idx, _shift, \ + _width, _min_mul, _max_mul, \ + _delay, _table), \ .common = { \ .regmap = NULL, \ .hw.init = CLK_HW_INIT_NO_PARENT(_name, \ @@ -78,7 +97,6 @@ struct owl_pll { } #define mul_mask(m) ((1 << ((m)->width)) - 1) -#define PLL_STABILITY_WAIT_US (50) static inline struct owl_pll *hw_to_owl_pll(const struct clk_hw *hw) { diff --git a/drivers/clk/actions/owl-s500.c b/drivers/clk/actions/owl-s500.c new file mode 100644 index 0000000000000..e2007ac4d235d --- /dev/null +++ b/drivers/clk/actions/owl-s500.c @@ -0,0 +1,525 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Actions Semi Owl S500 SoC clock driver + * + * Copyright (c) 2014 Actions Semi Inc. + * Author: David Liu <liuwei@actions-semi.com> + * + * Copyright (c) 2018 Linaro Ltd. + * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> + * + * Copyright (c) 2018 LSI-TEC - Caninos Loucos + * Author: Edgar Bernardi Righi <edgar.righi@lsitec.org.br> + */ + +#include <linux/clk-provider.h> +#include <linux/platform_device.h> + +#include "owl-common.h" +#include "owl-composite.h" +#include "owl-divider.h" +#include "owl-factor.h" +#include "owl-fixed-factor.h" +#include "owl-gate.h" +#include "owl-mux.h" +#include "owl-pll.h" + +#include <dt-bindings/clock/actions,s500-cmu.h> + +#define CMU_COREPLL (0x0000) +#define CMU_DEVPLL (0x0004) +#define CMU_DDRPLL (0x0008) +#define CMU_NANDPLL (0x000C) +#define CMU_DISPLAYPLL (0x0010) +#define CMU_AUDIOPLL (0x0014) +#define CMU_TVOUTPLL (0x0018) +#define CMU_BUSCLK (0x001C) +#define CMU_SENSORCLK (0x0020) +#define CMU_LCDCLK (0x0024) +#define CMU_DSICLK (0x0028) +#define CMU_CSICLK (0x002C) +#define CMU_DECLK (0x0030) +#define CMU_BISPCLK (0x0034) +#define CMU_BUSCLK1 (0x0038) +#define CMU_VDECLK (0x0040) +#define CMU_VCECLK (0x0044) +#define CMU_NANDCCLK (0x004C) +#define CMU_SD0CLK (0x0050) +#define CMU_SD1CLK (0x0054) +#define CMU_SD2CLK (0x0058) +#define CMU_UART0CLK (0x005C) +#define CMU_UART1CLK (0x0060) +#define CMU_UART2CLK (0x0064) +#define CMU_PWM4CLK (0x0068) +#define CMU_PWM5CLK (0x006C) +#define CMU_PWM0CLK (0x0070) +#define CMU_PWM1CLK (0x0074) +#define CMU_PWM2CLK (0x0078) +#define CMU_PWM3CLK (0x007C) +#define CMU_USBPLL (0x0080) +#define CMU_ETHERNETPLL (0x0084) +#define CMU_CVBSPLL (0x0088) +#define CMU_LENSCLK (0x008C) +#define CMU_GPU3DCLK (0x0090) +#define CMU_CORECTL (0x009C) +#define CMU_DEVCLKEN0 (0x00A0) +#define CMU_DEVCLKEN1 (0x00A4) +#define CMU_DEVRST0 (0x00A8) +#define CMU_DEVRST1 (0x00AC) +#define CMU_UART3CLK (0x00B0) +#define CMU_UART4CLK (0x00B4) +#define CMU_UART5CLK (0x00B8) +#define CMU_UART6CLK (0x00BC) +#define CMU_SSCLK (0x00C0) +#define CMU_DIGITALDEBUG (0x00D0) +#define CMU_ANALOGDEBUG (0x00D4) +#define CMU_COREPLLDEBUG (0x00D8) +#define CMU_DEVPLLDEBUG (0x00DC) +#define CMU_DDRPLLDEBUG (0x00E0) +#define CMU_NANDPLLDEBUG (0x00E4) +#define CMU_DISPLAYPLLDEBUG (0x00E8) +#define CMU_TVOUTPLLDEBUG (0x00EC) +#define CMU_DEEPCOLORPLLDEBUG (0x00F4) +#define CMU_AUDIOPLL_ETHPLLDEBUG (0x00F8) +#define CMU_CVBSPLLDEBUG (0x00FC) + +#define OWL_S500_COREPLL_DELAY (150) +#define OWL_S500_DDRPLL_DELAY (63) +#define OWL_S500_DEVPLL_DELAY (28) +#define OWL_S500_NANDPLL_DELAY (44) +#define OWL_S500_DISPLAYPLL_DELAY (57) +#define OWL_S500_ETHERNETPLL_DELAY (25) +#define OWL_S500_AUDIOPLL_DELAY (100) + +static const struct clk_pll_table clk_audio_pll_table[] = { + { 0, 45158400 }, { 1, 49152000 }, + { 0, 0 }, +}; + +/* pll clocks */ +static OWL_PLL_NO_PARENT_DELAY(ethernet_pll_clk, "ethernet_pll_clk", CMU_ETHERNETPLL, 500000000, 0, 0, 0, 0, 0, OWL_S500_ETHERNETPLL_DELAY, NULL, CLK_IGNORE_UNUSED); +static OWL_PLL_NO_PARENT_DELAY(core_pll_clk, "core_pll_clk", CMU_COREPLL, 12000000, 9, 0, 8, 4, 134, OWL_S500_COREPLL_DELAY, NULL, CLK_IGNORE_UNUSED); +static OWL_PLL_NO_PARENT_DELAY(ddr_pll_clk, "ddr_pll_clk", CMU_DDRPLL, 12000000, 8, 0, 8, 1, 67, OWL_S500_DDRPLL_DELAY, NULL, CLK_IGNORE_UNUSED); +static OWL_PLL_NO_PARENT_DELAY(nand_pll_clk, "nand_pll_clk", CMU_NANDPLL, 6000000, 8, 0, 7, 2, 86, OWL_S500_NANDPLL_DELAY, NULL, CLK_IGNORE_UNUSED); +static OWL_PLL_NO_PARENT_DELAY(display_pll_clk, "display_pll_clk", CMU_DISPLAYPLL, 6000000, 8, 0, 8, 2, 126, OWL_S500_DISPLAYPLL_DELAY, NULL, CLK_IGNORE_UNUSED); +static OWL_PLL_NO_PARENT_DELAY(dev_pll_clk, "dev_pll_clk", CMU_DEVPLL, 6000000, 8, 0, 7, 8, 126, OWL_S500_DEVPLL_DELAY, NULL, CLK_IGNORE_UNUSED); +static OWL_PLL_NO_PARENT_DELAY(audio_pll_clk, "audio_pll_clk", CMU_AUDIOPLL, 0, 4, 0, 1, 0, 0, OWL_S500_AUDIOPLL_DELAY, clk_audio_pll_table, CLK_IGNORE_UNUSED); + +static const char * const dev_clk_mux_p[] = { "hosc", "dev_pll_clk" }; +static const char * const bisp_clk_mux_p[] = { "display_pll_clk", "dev_clk" }; +static const char * const sensor_clk_mux_p[] = { "hosc", "bisp_clk" }; +static const char * const sd_clk_mux_p[] = { "dev_clk", "nand_pll_clk" }; +static const char * const pwm_clk_mux_p[] = { "losc", "hosc" }; +static const char * const ahbprediv_clk_mux_p[] = { "dev_clk", "display_pll_clk", "nand_pll_clk", "ddr_pll_clk" }; +static const char * const uart_clk_mux_p[] = { "hosc", "dev_pll_clk" }; +static const char * const de_clk_mux_p[] = { "display_pll_clk", "dev_clk" }; +static const char * const i2s_clk_mux_p[] = { "audio_pll_clk" }; +static const char * const hde_clk_mux_p[] = { "dev_clk", "display_pll_clk", "nand_pll_clk", "ddr_pll_clk" }; +static const char * const nand_clk_mux_p[] = { "nand_pll_clk", "display_pll_clk", "dev_clk", "ddr_pll_clk" }; + +static struct clk_factor_table sd_factor_table[] = { + /* bit0 ~ 4 */ + { 0, 1, 1 }, { 1, 1, 2 }, { 2, 1, 3 }, { 3, 1, 4 }, + { 4, 1, 5 }, { 5, 1, 6 }, { 6, 1, 7 }, { 7, 1, 8 }, + { 8, 1, 9 }, { 9, 1, 10 }, { 10, 1, 11 }, { 11, 1, 12 }, + { 12, 1, 13 }, { 13, 1, 14 }, { 14, 1, 15 }, { 15, 1, 16 }, + { 16, 1, 17 }, { 17, 1, 18 }, { 18, 1, 19 }, { 19, 1, 20 }, + { 20, 1, 21 }, { 21, 1, 22 }, { 22, 1, 23 }, { 23, 1, 24 }, + { 24, 1, 25 }, { 25, 1, 26 }, { 26, 1, 27 }, { 27, 1, 28 }, + { 28, 1, 29 }, { 29, 1, 30 }, { 30, 1, 31 }, { 31, 1, 32 }, + + /* bit8: /128 */ + { 256, 1, 1 * 128 }, { 257, 1, 2 * 128 }, { 258, 1, 3 * 128 }, { 259, 1, 4 * 128 }, + { 260, 1, 5 * 128 }, { 261, 1, 6 * 128 }, { 262, 1, 7 * 128 }, { 263, 1, 8 * 128 }, + { 264, 1, 9 * 128 }, { 265, 1, 10 * 128 }, { 266, 1, 11 * 128 }, { 267, 1, 12 * 128 }, + { 268, 1, 13 * 128 }, { 269, 1, 14 * 128 }, { 270, 1, 15 * 128 }, { 271, 1, 16 * 128 }, + { 272, 1, 17 * 128 }, { 273, 1, 18 * 128 }, { 274, 1, 19 * 128 }, { 275, 1, 20 * 128 }, + { 276, 1, 21 * 128 }, { 277, 1, 22 * 128 }, { 278, 1, 23 * 128 }, { 279, 1, 24 * 128 }, + { 280, 1, 25 * 128 }, { 281, 1, 26 * 128 }, { 282, 1, 27 * 128 }, { 283, 1, 28 * 128 }, + { 284, 1, 29 * 128 }, { 285, 1, 30 * 128 }, { 286, 1, 31 * 128 }, { 287, 1, 32 * 128 }, + { 0, 0, 0 }, +}; + +static struct clk_factor_table bisp_factor_table[] = { + { 0, 1, 1 }, { 1, 1, 2 }, { 2, 1, 3 }, { 3, 1, 4 }, + { 4, 1, 5 }, { 5, 1, 6 }, { 6, 1, 7 }, { 7, 1, 8 }, + { 0, 0, 0 }, +}; + +static struct clk_factor_table ahb_factor_table[] = { + { 1, 1, 2 }, { 2, 1, 3 }, + { 0, 0, 0 }, +}; + +static struct clk_div_table rmii_ref_div_table[] = { + { 0, 4 }, { 1, 10 }, + { 0, 0 }, +}; + +static struct clk_div_table i2s_div_table[] = { + { 0, 1 }, { 1, 2 }, { 2, 3 }, { 3, 4 }, + { 4, 6 }, { 5, 8 }, { 6, 12 }, { 7, 16 }, + { 8, 24 }, + { 0, 0 }, +}; + +static struct clk_div_table nand_div_table[] = { + { 0, 1 }, { 1, 2 }, { 2, 4 }, { 3, 6 }, + { 4, 8 }, { 5, 10 }, { 6, 12 }, { 7, 14 }, + { 8, 16 }, { 9, 18 }, { 10, 20 }, { 11, 22 }, + { 0, 0 }, +}; + +/* mux clock */ +static OWL_MUX(dev_clk, "dev_clk", dev_clk_mux_p, CMU_DEVPLL, 12, 1, CLK_SET_RATE_PARENT); +static OWL_MUX(ahbprediv_clk, "ahbprediv_clk", ahbprediv_clk_mux_p, CMU_BUSCLK1, 8, 3, CLK_SET_RATE_PARENT); + +/* gate clocks */ +static OWL_GATE(spi0_clk, "spi0_clk", "ahb_clk", CMU_DEVCLKEN1, 10, 0, CLK_IGNORE_UNUSED); +static OWL_GATE(spi1_clk, "spi1_clk", "ahb_clk", CMU_DEVCLKEN1, 11, 0, CLK_IGNORE_UNUSED); +static OWL_GATE(spi2_clk, "spi2_clk", "ahb_clk", CMU_DEVCLKEN1, 12, 0, CLK_IGNORE_UNUSED); +static OWL_GATE(spi3_clk, "spi3_clk", "ahb_clk", CMU_DEVCLKEN1, 13, 0, CLK_IGNORE_UNUSED); +static OWL_GATE(timer_clk, "timer_clk", "hosc", CMU_DEVCLKEN1, 27, 0, 0); +static OWL_GATE(hdmi_clk, "hdmi_clk", "hosc", CMU_DEVCLKEN1, 3, 0, 0); + +/* divider clocks */ +static OWL_DIVIDER(h_clk, "h_clk", "ahbprevdiv_clk", CMU_BUSCLK1, 12, 2, NULL, 0, 0); +static OWL_DIVIDER(rmii_ref_clk, "rmii_ref_clk", "ethernet_pll_clk", CMU_ETHERNETPLL, 1, 1, rmii_ref_div_table, 0, 0); + +/* factor clocks */ +static OWL_FACTOR(ahb_clk, "ahb_clk", "h_clk", CMU_BUSCLK1, 2, 2, ahb_factor_table, 0, 0); +static OWL_FACTOR(de1_clk, "de_clk1", "de_clk", CMU_DECLK, 0, 3, bisp_factor_table, 0, 0); +static OWL_FACTOR(de2_clk, "de_clk2", "de_clk", CMU_DECLK, 4, 3, bisp_factor_table, 0, 0); + +/* composite clocks */ +static OWL_COMP_FACTOR(vce_clk, "vce_clk", hde_clk_mux_p, + OWL_MUX_HW(CMU_VCECLK, 4, 2), + OWL_GATE_HW(CMU_DEVCLKEN0, 26, 0), + OWL_FACTOR_HW(CMU_VCECLK, 0, 3, 0, bisp_factor_table), + 0); + +static OWL_COMP_FACTOR(vde_clk, "vde_clk", hde_clk_mux_p, + OWL_MUX_HW(CMU_VDECLK, 4, 2), + OWL_GATE_HW(CMU_DEVCLKEN0, 25, 0), + OWL_FACTOR_HW(CMU_VDECLK, 0, 3, 0, bisp_factor_table), + 0); + +static OWL_COMP_FACTOR(bisp_clk, "bisp_clk", bisp_clk_mux_p, + OWL_MUX_HW(CMU_BISPCLK, 4, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 14, 0), + OWL_FACTOR_HW(CMU_BISPCLK, 0, 3, 0, bisp_factor_table), + 0); + +static OWL_COMP_FACTOR(sensor0_clk, "sensor0_clk", sensor_clk_mux_p, + OWL_MUX_HW(CMU_SENSORCLK, 4, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 14, 0), + OWL_FACTOR_HW(CMU_SENSORCLK, 0, 3, 0, bisp_factor_table), + CLK_IGNORE_UNUSED); + +static OWL_COMP_FACTOR(sensor1_clk, "sensor1_clk", sensor_clk_mux_p, + OWL_MUX_HW(CMU_SENSORCLK, 4, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 14, 0), + OWL_FACTOR_HW(CMU_SENSORCLK, 8, 3, 0, bisp_factor_table), + CLK_IGNORE_UNUSED); + +static OWL_COMP_FACTOR(sd0_clk, "sd0_clk", sd_clk_mux_p, + OWL_MUX_HW(CMU_SD0CLK, 9, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 5, 0), + OWL_FACTOR_HW(CMU_SD0CLK, 0, 9, 0, sd_factor_table), + 0); + +static OWL_COMP_FACTOR(sd1_clk, "sd1_clk", sd_clk_mux_p, + OWL_MUX_HW(CMU_SD1CLK, 9, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 6, 0), + OWL_FACTOR_HW(CMU_SD1CLK, 0, 9, 0, sd_factor_table), + 0); + +static OWL_COMP_FACTOR(sd2_clk, "sd2_clk", sd_clk_mux_p, + OWL_MUX_HW(CMU_SD2CLK, 9, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 7, 0), + OWL_FACTOR_HW(CMU_SD2CLK, 0, 9, 0, sd_factor_table), + 0); + +static OWL_COMP_DIV(pwm0_clk, "pwm0_clk", pwm_clk_mux_p, + OWL_MUX_HW(CMU_PWM0CLK, 12, 1), + OWL_GATE_HW(CMU_DEVCLKEN1, 23, 0), + OWL_DIVIDER_HW(CMU_PWM0CLK, 0, 10, 0, NULL), + 0); + +static OWL_COMP_DIV(pwm1_clk, "pwm1_clk", pwm_clk_mux_p, + OWL_MUX_HW(CMU_PWM1CLK, 12, 1), + OWL_GATE_HW(CMU_DEVCLKEN1, 24, 0), + OWL_DIVIDER_HW(CMU_PWM1CLK, 0, 10, 0, NULL), + 0); + +static OWL_COMP_DIV(pwm2_clk, "pwm2_clk", pwm_clk_mux_p, + OWL_MUX_HW(CMU_PWM2CLK, 12, 1), + OWL_GATE_HW(CMU_DEVCLKEN1, 25, 0), + OWL_DIVIDER_HW(CMU_PWM2CLK, 0, 10, 0, NULL), + 0); + +static OWL_COMP_DIV(pwm3_clk, "pwm3_clk", pwm_clk_mux_p, + OWL_MUX_HW(CMU_PWM3CLK, 12, 1), + OWL_GATE_HW(CMU_DEVCLKEN1, 26, 0), + OWL_DIVIDER_HW(CMU_PWM3CLK, 0, 10, 0, NULL), + 0); + +static OWL_COMP_DIV(pwm4_clk, "pwm4_clk", pwm_clk_mux_p, + OWL_MUX_HW(CMU_PWM4CLK, 12, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 11, 0), + OWL_DIVIDER_HW(CMU_PWM4CLK, 0, 10, 0, NULL), + 0); + +static OWL_COMP_DIV(pwm5_clk, "pwm5_clk", pwm_clk_mux_p, + OWL_MUX_HW(CMU_PWM5CLK, 12, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 0, 0), + OWL_DIVIDER_HW(CMU_PWM5CLK, 0, 10, 0, NULL), + 0); + +static OWL_COMP_PASS(de_clk, "de_clk", de_clk_mux_p, + OWL_MUX_HW(CMU_DECLK, 12, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 8, 0), + 0); + +static OWL_COMP_FIXED_FACTOR(i2c0_clk, "i2c0_clk", "ethernet_pll_clk", + OWL_GATE_HW(CMU_DEVCLKEN1, 14, 0), + 1, 5, 0); + +static OWL_COMP_FIXED_FACTOR(i2c1_clk, "i2c1_clk", "ethernet_pll_clk", + OWL_GATE_HW(CMU_DEVCLKEN1, 15, 0), + 1, 5, 0); + +static OWL_COMP_FIXED_FACTOR(i2c2_clk, "i2c2_clk", "ethernet_pll_clk", + OWL_GATE_HW(CMU_DEVCLKEN1, 30, 0), + 1, 5, 0); + +static OWL_COMP_FIXED_FACTOR(i2c3_clk, "i2c3_clk", "ethernet_pll_clk", + OWL_GATE_HW(CMU_DEVCLKEN1, 31, 0), + 1, 5, 0); + +static OWL_COMP_DIV(uart0_clk, "uart0_clk", uart_clk_mux_p, + OWL_MUX_HW(CMU_UART0CLK, 16, 1), + OWL_GATE_HW(CMU_DEVCLKEN1, 6, 0), + OWL_DIVIDER_HW(CMU_UART1CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL), + CLK_IGNORE_UNUSED); + +static OWL_COMP_DIV(uart1_clk, "uart1_clk", uart_clk_mux_p, + OWL_MUX_HW(CMU_UART1CLK, 16, 1), + OWL_GATE_HW(CMU_DEVCLKEN1, 7, 0), + OWL_DIVIDER_HW(CMU_UART1CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL), + CLK_IGNORE_UNUSED); + +static OWL_COMP_DIV(uart2_clk, "uart2_clk", uart_clk_mux_p, + OWL_MUX_HW(CMU_UART2CLK, 16, 1), + OWL_GATE_HW(CMU_DEVCLKEN1, 8, 0), + OWL_DIVIDER_HW(CMU_UART1CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL), + CLK_IGNORE_UNUSED); + +static OWL_COMP_DIV(uart3_clk, "uart3_clk", uart_clk_mux_p, + OWL_MUX_HW(CMU_UART3CLK, 16, 1), + OWL_GATE_HW(CMU_DEVCLKEN1, 19, 0), + OWL_DIVIDER_HW(CMU_UART1CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL), + CLK_IGNORE_UNUSED); + +static OWL_COMP_DIV(uart4_clk, "uart4_clk", uart_clk_mux_p, + OWL_MUX_HW(CMU_UART4CLK, 16, 1), + OWL_GATE_HW(CMU_DEVCLKEN1, 20, 0), + OWL_DIVIDER_HW(CMU_UART1CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL), + CLK_IGNORE_UNUSED); + +static OWL_COMP_DIV(uart5_clk, "uart5_clk", uart_clk_mux_p, + OWL_MUX_HW(CMU_UART5CLK, 16, 1), + OWL_GATE_HW(CMU_DEVCLKEN1, 21, 0), + OWL_DIVIDER_HW(CMU_UART1CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL), + CLK_IGNORE_UNUSED); + +static OWL_COMP_DIV(uart6_clk, "uart6_clk", uart_clk_mux_p, + OWL_MUX_HW(CMU_UART6CLK, 16, 1), + OWL_GATE_HW(CMU_DEVCLKEN1, 18, 0), + OWL_DIVIDER_HW(CMU_UART1CLK, 0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL), + CLK_IGNORE_UNUSED); + +static OWL_COMP_DIV(i2srx_clk, "i2srx_clk", i2s_clk_mux_p, + OWL_MUX_HW(CMU_AUDIOPLL, 24, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 21, 0), + OWL_DIVIDER_HW(CMU_AUDIOPLL, 20, 4, 0, i2s_div_table), + 0); + +static OWL_COMP_DIV(i2stx_clk, "i2stx_clk", i2s_clk_mux_p, + OWL_MUX_HW(CMU_AUDIOPLL, 24, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 20, 0), + OWL_DIVIDER_HW(CMU_AUDIOPLL, 16, 4, 0, i2s_div_table), + 0); + +static OWL_COMP_DIV(hdmia_clk, "hdmia_clk", i2s_clk_mux_p, + OWL_MUX_HW(CMU_AUDIOPLL, 24, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 22, 0), + OWL_DIVIDER_HW(CMU_AUDIOPLL, 24, 4, 0, i2s_div_table), + 0); + +static OWL_COMP_DIV(spdif_clk, "spdif_clk", i2s_clk_mux_p, + OWL_MUX_HW(CMU_AUDIOPLL, 24, 1), + OWL_GATE_HW(CMU_DEVCLKEN0, 23, 0), + OWL_DIVIDER_HW(CMU_AUDIOPLL, 28, 4, 0, i2s_div_table), + 0); + +static OWL_COMP_DIV(nand_clk, "nand_clk", nand_clk_mux_p, + OWL_MUX_HW(CMU_NANDCCLK, 8, 2), + OWL_GATE_HW(CMU_DEVCLKEN0, 4, 0), + OWL_DIVIDER_HW(CMU_NANDCCLK, 0, 3, 0, nand_div_table), + CLK_SET_RATE_PARENT); + +static OWL_COMP_DIV(ecc_clk, "ecc_clk", nand_clk_mux_p, + OWL_MUX_HW(CMU_NANDCCLK, 8, 2), + OWL_GATE_HW(CMU_DEVCLKEN0, 4, 0), + OWL_DIVIDER_HW(CMU_NANDCCLK, 4, 3, 0, nand_div_table), + CLK_SET_RATE_PARENT); + +static struct owl_clk_common *s500_clks[] = { + ðernet_pll_clk.common, + &core_pll_clk.common, + &ddr_pll_clk.common, + &dev_pll_clk.common, + &nand_pll_clk.common, + &audio_pll_clk.common, + &display_pll_clk.common, + &dev_clk.common, + &timer_clk.common, + &i2c0_clk.common, + &i2c1_clk.common, + &i2c2_clk.common, + &i2c3_clk.common, + &uart0_clk.common, + &uart1_clk.common, + &uart2_clk.common, + &uart3_clk.common, + &uart4_clk.common, + &uart5_clk.common, + &uart6_clk.common, + &pwm0_clk.common, + &pwm1_clk.common, + &pwm2_clk.common, + &pwm3_clk.common, + &pwm4_clk.common, + &pwm5_clk.common, + &sensor0_clk.common, + &sensor1_clk.common, + &sd0_clk.common, + &sd1_clk.common, + &sd2_clk.common, + &bisp_clk.common, + &ahb_clk.common, + &ahbprediv_clk.common, + &h_clk.common, + &spi0_clk.common, + &spi1_clk.common, + &spi2_clk.common, + &spi3_clk.common, + &rmii_ref_clk.common, + &de_clk.common, + &de1_clk.common, + &de2_clk.common, + &i2srx_clk.common, + &i2stx_clk.common, + &hdmia_clk.common, + &hdmi_clk.common, + &vce_clk.common, + &vde_clk.common, + &spdif_clk.common, + &nand_clk.common, + &ecc_clk.common, +}; + +static struct clk_hw_onecell_data s500_hw_clks = { + .hws = { + [CLK_ETHERNET_PLL] = ðernet_pll_clk.common.hw, + [CLK_CORE_PLL] = &core_pll_clk.common.hw, + [CLK_DDR_PLL] = &ddr_pll_clk.common.hw, + [CLK_NAND_PLL] = &nand_pll_clk.common.hw, + [CLK_DISPLAY_PLL] = &display_pll_clk.common.hw, + [CLK_DEV_PLL] = &dev_pll_clk.common.hw, + [CLK_AUDIO_PLL] = &audio_pll_clk.common.hw, + [CLK_TIMER] = &timer_clk.common.hw, + [CLK_DEV] = &dev_clk.common.hw, + [CLK_DE] = &de_clk.common.hw, + [CLK_DE1] = &de1_clk.common.hw, + [CLK_DE2] = &de2_clk.common.hw, + [CLK_I2C0] = &i2c0_clk.common.hw, + [CLK_I2C1] = &i2c1_clk.common.hw, + [CLK_I2C2] = &i2c2_clk.common.hw, + [CLK_I2C3] = &i2c3_clk.common.hw, + [CLK_I2SRX] = &i2srx_clk.common.hw, + [CLK_I2STX] = &i2stx_clk.common.hw, + [CLK_UART0] = &uart0_clk.common.hw, + [CLK_UART1] = &uart1_clk.common.hw, + [CLK_UART2] = &uart2_clk.common.hw, + [CLK_UART3] = &uart3_clk.common.hw, + [CLK_UART4] = &uart4_clk.common.hw, + [CLK_UART5] = &uart5_clk.common.hw, + [CLK_UART6] = &uart6_clk.common.hw, + [CLK_PWM0] = &pwm0_clk.common.hw, + [CLK_PWM1] = &pwm1_clk.common.hw, + [CLK_PWM2] = &pwm2_clk.common.hw, + [CLK_PWM3] = &pwm3_clk.common.hw, + [CLK_PWM4] = &pwm4_clk.common.hw, + [CLK_PWM5] = &pwm5_clk.common.hw, + [CLK_SENSOR0] = &sensor0_clk.common.hw, + [CLK_SENSOR1] = &sensor1_clk.common.hw, + [CLK_SD0] = &sd0_clk.common.hw, + [CLK_SD1] = &sd1_clk.common.hw, + [CLK_SD2] = &sd2_clk.common.hw, + [CLK_BISP] = &bisp_clk.common.hw, + [CLK_SPI0] = &spi0_clk.common.hw, + [CLK_SPI1] = &spi1_clk.common.hw, + [CLK_SPI2] = &spi2_clk.common.hw, + [CLK_SPI3] = &spi3_clk.common.hw, + [CLK_AHB] = &ahb_clk.common.hw, + [CLK_H] = &h_clk.common.hw, + [CLK_AHBPREDIV] = &ahbprediv_clk.common.hw, + [CLK_RMII_REF] = &rmii_ref_clk.common.hw, + [CLK_HDMI_AUDIO] = &hdmia_clk.common.hw, + [CLK_HDMI] = &hdmi_clk.common.hw, + [CLK_VDE] = &vde_clk.common.hw, + [CLK_VCE] = &vce_clk.common.hw, + [CLK_SPDIF] = &spdif_clk.common.hw, + [CLK_NAND] = &nand_clk.common.hw, + [CLK_ECC] = &ecc_clk.common.hw, + }, + .num = CLK_NR_CLKS, +}; + +static struct owl_clk_desc s500_clk_desc = { + .clks = s500_clks, + .num_clks = ARRAY_SIZE(s500_clks), + + .hw_clks = &s500_hw_clks, +}; + +static int s500_clk_probe(struct platform_device *pdev) +{ + struct owl_clk_desc *desc; + + desc = &s500_clk_desc; + owl_clk_regmap_init(pdev, desc); + + return owl_clk_probe(&pdev->dev, desc->hw_clks); +} + +static const struct of_device_id s500_clk_of_match[] = { + { .compatible = "actions,s500-cmu", }, + { /* sentinel */ } +}; + +static struct platform_driver s500_clk_driver = { + .probe = s500_clk_probe, + .driver = { + .name = "s500-cmu", + .of_match_table = s500_clk_of_match, + }, +}; + +static int __init s500_clk_init(void) +{ + return platform_driver_register(&s500_clk_driver); +} +core_initcall(s500_clk_init); diff --git a/drivers/clk/at91/clk-audio-pll.c b/drivers/clk/at91/clk-audio-pll.c index 36d77146a3bd4..3cc4a82f4e9fb 100644 --- a/drivers/clk/at91/clk-audio-pll.c +++ b/drivers/clk/at91/clk-audio-pll.c @@ -340,7 +340,12 @@ static long clk_audio_pll_pmc_round_rate(struct clk_hw *hw, unsigned long rate, pr_debug("A PLL/PMC: %s, rate = %lu (parent_rate = %lu)\n", __func__, rate, *parent_rate); - for (div = 1; div <= AUDIO_PLL_QDPMC_MAX; div++) { + if (!rate) + return 0; + + best_parent_rate = clk_round_rate(pclk->clk, 1); + div = max(best_parent_rate / rate, 1UL); + for (; div <= AUDIO_PLL_QDPMC_MAX; div++) { best_parent_rate = clk_round_rate(pclk->clk, rate * div); tmp_rate = best_parent_rate / div; tmp_diff = abs(rate - tmp_rate); @@ -350,6 +355,8 @@ static long clk_audio_pll_pmc_round_rate(struct clk_hw *hw, unsigned long rate, best_rate = tmp_rate; best_diff = tmp_diff; tmp_qd = div; + if (!best_diff) + break; /* got exact match */ } } diff --git a/drivers/clk/at91/clk-programmable.c b/drivers/clk/at91/clk-programmable.c index 5bc68b9c54984..89d6f3736dbf6 100644 --- a/drivers/clk/at91/clk-programmable.c +++ b/drivers/clk/at91/clk-programmable.c @@ -132,11 +132,8 @@ static int clk_programmable_set_rate(struct clk_hw *hw, unsigned long rate, struct clk_programmable *prog = to_clk_programmable(hw); const struct clk_programmable_layout *layout = prog->layout; unsigned long div = parent_rate / rate; - unsigned int pckr; int shift = 0; - regmap_read(prog->regmap, AT91_PMC_PCKR(prog->id), &pckr); - if (!div) return -EINVAL; diff --git a/drivers/clk/at91/sama5d2.c b/drivers/clk/at91/sama5d2.c index cd0ef7274fdbf..1f70cb164b06f 100644 --- a/drivers/clk/at91/sama5d2.c +++ b/drivers/clk/at91/sama5d2.c @@ -241,13 +241,14 @@ static void __init sama5d2_pmc_setup(struct device_node *np) parent_names[2] = "plladivck"; parent_names[3] = "utmick"; parent_names[4] = "masterck"; + parent_names[5] = "audiopll_pmcck"; for (i = 0; i < 3; i++) { char name[6]; snprintf(name, sizeof(name), "prog%d", i); hw = at91_clk_register_programmable(regmap, name, - parent_names, 5, i, + parent_names, 6, i, &at91sam9x5_programmable_layout); if (IS_ERR(hw)) goto err_free; diff --git a/drivers/clk/clk-clps711x.c b/drivers/clk/clk-clps711x.c index 2c04396402ab4..c36c47bdba025 100644 --- a/drivers/clk/clk-clps711x.c +++ b/drivers/clk/clk-clps711x.c @@ -44,21 +44,21 @@ struct clps711x_clk { struct clk_hw_onecell_data clk_data; }; -static struct clps711x_clk * __init _clps711x_clk_init(void __iomem *base, - u32 fref) +static void __init clps711x_clk_init_dt(struct device_node *np) { - u32 tmp, f_cpu, f_pll, f_bus, f_tim, f_pwm, f_spi; + u32 tmp, f_cpu, f_pll, f_bus, f_tim, f_pwm, f_spi, fref = 0; struct clps711x_clk *clps711x_clk; - unsigned i; + void __iomem *base; + + WARN_ON(of_property_read_u32(np, "startup-frequency", &fref)); - if (!base) - return ERR_PTR(-ENOMEM); + base = of_iomap(np, 0); + BUG_ON(!base); clps711x_clk = kzalloc(struct_size(clps711x_clk, clk_data.hws, CLPS711X_CLK_MAX), GFP_KERNEL); - if (!clps711x_clk) - return ERR_PTR(-ENOMEM); + BUG_ON(!clps711x_clk); spin_lock_init(&clps711x_clk->lock); @@ -137,52 +137,13 @@ static struct clps711x_clk * __init _clps711x_clk_init(void __iomem *base, clk_hw_register_fixed_factor(NULL, "uart", "bus", 0, 1, 10); clps711x_clk->clk_data.hws[CLPS711X_CLK_TICK] = clk_hw_register_fixed_rate(NULL, "tick", NULL, 0, 64); - for (i = 0; i < CLPS711X_CLK_MAX; i++) - if (IS_ERR(clps711x_clk->clk_data.hws[i])) + for (tmp = 0; tmp < CLPS711X_CLK_MAX; tmp++) + if (IS_ERR(clps711x_clk->clk_data.hws[tmp])) pr_err("clk %i: register failed with %ld\n", - i, PTR_ERR(clps711x_clk->clk_data.hws[i])); - - return clps711x_clk; -} - -void __init clps711x_clk_init(void __iomem *base) -{ - struct clps711x_clk *clps711x_clk; - - clps711x_clk = _clps711x_clk_init(base, 73728000); - - BUG_ON(IS_ERR(clps711x_clk)); - - /* Clocksource */ - clk_hw_register_clkdev(clps711x_clk->clk_data.hws[CLPS711X_CLK_TIMER1], - NULL, "clps711x-timer.0"); - clk_hw_register_clkdev(clps711x_clk->clk_data.hws[CLPS711X_CLK_TIMER2], - NULL, "clps711x-timer.1"); - - /* Drivers */ - clk_hw_register_clkdev(clps711x_clk->clk_data.hws[CLPS711X_CLK_PWM], - NULL, "clps711x-pwm"); - clk_hw_register_clkdev(clps711x_clk->clk_data.hws[CLPS711X_CLK_UART], - NULL, "clps711x-uart.0"); - clk_hw_register_clkdev(clps711x_clk->clk_data.hws[CLPS711X_CLK_UART], - NULL, "clps711x-uart.1"); -} - -#ifdef CONFIG_OF -static void __init clps711x_clk_init_dt(struct device_node *np) -{ - void __iomem *base = of_iomap(np, 0); - struct clps711x_clk *clps711x_clk; - u32 fref = 0; - - WARN_ON(of_property_read_u32(np, "startup-frequency", &fref)); - - clps711x_clk = _clps711x_clk_init(base, fref); - BUG_ON(IS_ERR(clps711x_clk)); + tmp, PTR_ERR(clps711x_clk->clk_data.hws[tmp])); clps711x_clk->clk_data.num = CLPS711X_CLK_MAX; of_clk_add_hw_provider(np, of_clk_hw_onecell_get, &clps711x_clk->clk_data); } CLK_OF_DECLARE(clps711x, "cirrus,ep7209-clk", clps711x_clk_init_dt); -#endif diff --git a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c index c9a86156ced85..daa1fc8fba537 100644 --- a/drivers/clk/clk-devres.c +++ b/drivers/clk/clk-devres.c @@ -29,6 +29,17 @@ struct clk *devm_clk_get(struct device *dev, const char *id) } EXPORT_SYMBOL(devm_clk_get); +struct clk *devm_clk_get_optional(struct device *dev, const char *id) +{ + struct clk *clk = devm_clk_get(dev, id); + + if (clk == ERR_PTR(-ENOENT)) + return NULL; + + return clk; +} +EXPORT_SYMBOL(devm_clk_get_optional); + struct clk_bulk_devres { struct clk_bulk_data *clks; int num_clks; diff --git a/drivers/clk/clk-fixed-mmio.c b/drivers/clk/clk-fixed-mmio.c new file mode 100644 index 0000000000000..d1a97d971183e --- /dev/null +++ b/drivers/clk/clk-fixed-mmio.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Memory Mapped IO Fixed clock driver + * + * Copyright (C) 2018 Cadence Design Systems, Inc. + * + * Authors: + * Jan Kotas <jank@cadence.com> + */ + +#include <linux/clk-provider.h> +#include <linux/of_address.h> +#include <linux/module.h> +#include <linux/platform_device.h> + +static struct clk_hw *fixed_mmio_clk_setup(struct device_node *node) +{ + struct clk_hw *clk; + const char *clk_name = node->name; + void __iomem *base; + u32 freq; + int ret; + + base = of_iomap(node, 0); + if (!base) { + pr_err("%pOFn: failed to map address\n", node); + return ERR_PTR(-EIO); + } + + freq = readl(base); + iounmap(base); + of_property_read_string(node, "clock-output-names", &clk_name); + + clk = clk_hw_register_fixed_rate(NULL, clk_name, NULL, 0, freq); + if (IS_ERR(clk)) { + pr_err("%pOFn: failed to register fixed rate clock\n", node); + return clk; + } + + ret = of_clk_add_hw_provider(node, of_clk_hw_simple_get, clk); + if (ret) { + pr_err("%pOFn: failed to add clock provider\n", node); + clk_hw_unregister(clk); + clk = ERR_PTR(ret); + } + + return clk; +} + +static void __init of_fixed_mmio_clk_setup(struct device_node *node) +{ + fixed_mmio_clk_setup(node); +} +CLK_OF_DECLARE(fixed_mmio_clk, "fixed-mmio-clock", of_fixed_mmio_clk_setup); + +/** + * This is not executed when of_fixed_mmio_clk_setup succeeded. + */ +static int of_fixed_mmio_clk_probe(struct platform_device *pdev) +{ + struct clk_hw *clk; + + clk = fixed_mmio_clk_setup(pdev->dev.of_node); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + platform_set_drvdata(pdev, clk); + + return 0; +} + +static int of_fixed_mmio_clk_remove(struct platform_device *pdev) +{ + struct clk_hw *clk = platform_get_drvdata(pdev); + + of_clk_del_provider(pdev->dev.of_node); + clk_hw_unregister_fixed_rate(clk); + + return 0; +} + +static const struct of_device_id of_fixed_mmio_clk_ids[] = { + { .compatible = "fixed-mmio-clock" }, + { } +}; +MODULE_DEVICE_TABLE(of, of_fixed_mmio_clk_ids); + +static struct platform_driver of_fixed_mmio_clk_driver = { + .driver = { + .name = "of_fixed_mmio_clk", + .of_match_table = of_fixed_mmio_clk_ids, + }, + .probe = of_fixed_mmio_clk_probe, + .remove = of_fixed_mmio_clk_remove, +}; +module_platform_driver(of_fixed_mmio_clk_driver); + +MODULE_AUTHOR("Jan Kotas <jank@cadence.com>"); +MODULE_DESCRIPTION("Memory Mapped IO Fixed clock driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/clk-fractional-divider.c b/drivers/clk/clk-fractional-divider.c index 545dceec0bbf0..fdfe2e423d150 100644 --- a/drivers/clk/clk-fractional-divider.c +++ b/drivers/clk/clk-fractional-divider.c @@ -79,7 +79,7 @@ static long clk_fd_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long m, n; u64 ret; - if (!rate || rate >= *parent_rate) + if (!rate || (!clk_hw_can_set_rate_parent(hw) && rate >= *parent_rate)) return *parent_rate; if (fd->approximation) diff --git a/drivers/clk/clk-gpio.c b/drivers/clk/clk-gpio.c index 25eed3e0251f3..c2f07f0d077c7 100644 --- a/drivers/clk/clk-gpio.c +++ b/drivers/clk/clk-gpio.c @@ -58,6 +58,35 @@ const struct clk_ops clk_gpio_gate_ops = { }; EXPORT_SYMBOL_GPL(clk_gpio_gate_ops); +static int clk_sleeping_gpio_gate_prepare(struct clk_hw *hw) +{ + struct clk_gpio *clk = to_clk_gpio(hw); + + gpiod_set_value_cansleep(clk->gpiod, 1); + + return 0; +} + +static void clk_sleeping_gpio_gate_unprepare(struct clk_hw *hw) +{ + struct clk_gpio *clk = to_clk_gpio(hw); + + gpiod_set_value_cansleep(clk->gpiod, 0); +} + +static int clk_sleeping_gpio_gate_is_prepared(struct clk_hw *hw) +{ + struct clk_gpio *clk = to_clk_gpio(hw); + + return gpiod_get_value_cansleep(clk->gpiod); +} + +static const struct clk_ops clk_sleeping_gpio_gate_ops = { + .prepare = clk_sleeping_gpio_gate_prepare, + .unprepare = clk_sleeping_gpio_gate_unprepare, + .is_prepared = clk_sleeping_gpio_gate_is_prepared, +}; + /** * DOC: basic clock multiplexer which can be controlled with a gpio output * Traits of this clock: @@ -144,10 +173,16 @@ struct clk_hw *clk_hw_register_gpio_gate(struct device *dev, const char *name, const char *parent_name, struct gpio_desc *gpiod, unsigned long flags) { + const struct clk_ops *ops; + + if (gpiod_cansleep(gpiod)) + ops = &clk_sleeping_gpio_gate_ops; + else + ops = &clk_gpio_gate_ops; + return clk_register_gpio(dev, name, (parent_name ? &parent_name : NULL), - (parent_name ? 1 : 0), gpiod, flags, - &clk_gpio_gate_ops); + (parent_name ? 1 : 0), gpiod, flags, ops); } EXPORT_SYMBOL_GPL(clk_hw_register_gpio_gate); diff --git a/drivers/clk/clk-highbank.c b/drivers/clk/clk-highbank.c index 727ed8e1bb726..8e4581004695c 100644 --- a/drivers/clk/clk-highbank.c +++ b/drivers/clk/clk-highbank.c @@ -293,6 +293,7 @@ static __init struct clk *hb_clk_init(struct device_node *node, const struct clk /* Map system registers */ srnp = of_find_compatible_node(NULL, NULL, "calxeda,hb-sregs"); hb_clk->reg = of_iomap(srnp, 0); + of_node_put(srnp); BUG_ON(!hb_clk->reg); hb_clk->reg += reg; diff --git a/drivers/clk/clk-max77686.c b/drivers/clk/clk-max77686.c index 22c937644c93e..3727d54724500 100644 --- a/drivers/clk/clk-max77686.c +++ b/drivers/clk/clk-max77686.c @@ -235,8 +235,9 @@ static int max77686_clk_probe(struct platform_device *pdev) return ret; } - ret = clk_hw_register_clkdev(&max_clk_data->hw, - max_clk_data->clk_idata.name, NULL); + ret = devm_clk_hw_register_clkdev(dev, &max_clk_data->hw, + max_clk_data->clk_idata.name, + NULL); if (ret < 0) { dev_err(dev, "Failed to clkdev register: %d\n", ret); return ret; @@ -244,8 +245,8 @@ static int max77686_clk_probe(struct platform_device *pdev) } if (parent->of_node) { - ret = of_clk_add_hw_provider(parent->of_node, of_clk_max77686_get, - drv_data); + ret = devm_of_clk_add_hw_provider(dev, of_clk_max77686_get, + drv_data); if (ret < 0) { dev_err(dev, "Failed to register OF clock provider: %d\n", @@ -261,27 +262,11 @@ static int max77686_clk_probe(struct platform_device *pdev) 1 << MAX77802_CLOCK_LOW_JITTER_SHIFT); if (ret < 0) { dev_err(dev, "Failed to config low-jitter: %d\n", ret); - goto remove_of_clk_provider; + return ret; } } return 0; - -remove_of_clk_provider: - if (parent->of_node) - of_clk_del_provider(parent->of_node); - - return ret; -} - -static int max77686_clk_remove(struct platform_device *pdev) -{ - struct device *parent = pdev->dev.parent; - - if (parent->of_node) - of_clk_del_provider(parent->of_node); - - return 0; } static const struct platform_device_id max77686_clk_id[] = { @@ -297,7 +282,6 @@ static struct platform_driver max77686_clk_driver = { .name = "max77686-clk", }, .probe = max77686_clk_probe, - .remove = max77686_clk_remove, .id_table = max77686_clk_id, }; diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c index 5baa9e051110c..1212a9be7e80f 100644 --- a/drivers/clk/clk-qoriq.c +++ b/drivers/clk/clk-qoriq.c @@ -1148,8 +1148,8 @@ static void __init create_one_pll(struct clockgen *cg, int idx) pll->div[i].clk = clk; ret = clk_register_clkdev(clk, pll->div[i].name, NULL); if (ret != 0) - pr_err("%s: %s: register to lookup table failed %ld\n", - __func__, pll->div[i].name, PTR_ERR(clk)); + pr_err("%s: %s: register to lookup table failed %d\n", + __func__, pll->div[i].name, ret); } } @@ -1389,6 +1389,7 @@ static void __init clockgen_init(struct device_node *np) pr_err("%s: Couldn't map %pOF regs\n", __func__, guts); } + of_node_put(guts); } } diff --git a/drivers/clk/clk-stm32mp1.c b/drivers/clk/clk-stm32mp1.c index 6a31f7f434ce4..a0ae8dc169090 100644 --- a/drivers/clk/clk-stm32mp1.c +++ b/drivers/clk/clk-stm32mp1.c @@ -121,7 +121,7 @@ static const char * const cpu_src[] = { }; static const char * const axi_src[] = { - "ck_hsi", "ck_hse", "pll2_p", "pll3_p" + "ck_hsi", "ck_hse", "pll2_p" }; static const char * const per_src[] = { @@ -225,19 +225,19 @@ static const char * const usart6_src[] = { }; static const char * const fdcan_src[] = { - "ck_hse", "pll3_q", "pll4_q" + "ck_hse", "pll3_q", "pll4_q", "pll4_r" }; static const char * const sai_src[] = { - "pll4_q", "pll3_q", "i2s_ckin", "ck_per" + "pll4_q", "pll3_q", "i2s_ckin", "ck_per", "pll3_r" }; static const char * const sai2_src[] = { - "pll4_q", "pll3_q", "i2s_ckin", "ck_per", "spdif_ck_symb" + "pll4_q", "pll3_q", "i2s_ckin", "ck_per", "spdif_ck_symb", "pll3_r" }; static const char * const adc12_src[] = { - "pll4_q", "ck_per" + "pll4_r", "ck_per", "pll3_q" }; static const char * const dsi_src[] = { @@ -269,7 +269,7 @@ static const struct clk_div_table axi_div_table[] = { static const struct clk_div_table mcu_div_table[] = { { 0, 1 }, { 1, 2 }, { 2, 4 }, { 3, 8 }, { 4, 16 }, { 5, 32 }, { 6, 64 }, { 7, 128 }, - { 8, 512 }, { 9, 512 }, { 10, 512}, { 11, 512 }, + { 8, 256 }, { 9, 512 }, { 10, 512}, { 11, 512 }, { 12, 512 }, { 13, 512 }, { 14, 512}, { 15, 512 }, { 0 }, }; @@ -1286,10 +1286,11 @@ _clk_stm32_register_composite(struct device *dev, MGATE_MP1(_id, _name, _parent, _flags, _mgate) #define KCLK(_id, _name, _parents, _flags, _mgate, _mmux)\ - COMPOSITE(_id, _name, _parents, CLK_OPS_PARENT_ENABLE | _flags,\ - _MGATE_MP1(_mgate),\ - _MMUX(_mmux),\ - _NO_DIV) + COMPOSITE(_id, _name, _parents, CLK_OPS_PARENT_ENABLE |\ + CLK_SET_RATE_NO_REPARENT | _flags,\ + _MGATE_MP1(_mgate),\ + _MMUX(_mmux),\ + _NO_DIV) enum { G_SAI1, @@ -1655,12 +1656,14 @@ static const struct stm32_mux_cfg ker_mux_cfg[M_LAST] = { static const struct clock_config stm32mp1_clock_cfg[] = { /* Oscillator divider */ - DIV(NO_ID, "clk-hsi-div", "clk-hsi", 0, RCC_HSICFGR, 0, 2, - CLK_DIVIDER_READ_ONLY), + DIV(NO_ID, "clk-hsi-div", "clk-hsi", CLK_DIVIDER_POWER_OF_TWO, + RCC_HSICFGR, 0, 2, CLK_DIVIDER_READ_ONLY), /* External / Internal Oscillators */ GATE_MP1(CK_HSE, "ck_hse", "clk-hse", 0, RCC_OCENSETR, 8, 0), - GATE_MP1(CK_CSI, "ck_csi", "clk-csi", 0, RCC_OCENSETR, 4, 0), + /* ck_csi is used by IO compensation and should be critical */ + GATE_MP1(CK_CSI, "ck_csi", "clk-csi", CLK_IS_CRITICAL, + RCC_OCENSETR, 4, 0), GATE_MP1(CK_HSI, "ck_hsi", "clk-hsi-div", 0, RCC_OCENSETR, 0, 0), GATE(CK_LSI, "ck_lsi", "clk-lsi", 0, RCC_RDLSICR, 0, 0), GATE(CK_LSE, "ck_lse", "clk-lse", 0, RCC_BDCR, 0, 0), @@ -1952,14 +1955,14 @@ static const struct clock_config stm32mp1_clock_cfg[] = { MGATE_MP1(GPU_K, "gpu_k", "pll2_q", 0, G_GPU), MGATE_MP1(DAC12_K, "dac12_k", "ck_lsi", 0, G_DAC12), - COMPOSITE(ETHPTP_K, "ethptp_k", eth_src, CLK_OPS_PARENT_ENABLE, + COMPOSITE(ETHPTP_K, "ethptp_k", eth_src, CLK_OPS_PARENT_ENABLE | + CLK_SET_RATE_NO_REPARENT, _NO_GATE, _MMUX(M_ETHCK), - _DIV(RCC_ETHCKSELR, 4, 4, CLK_DIVIDER_ALLOW_ZERO, NULL)), + _DIV(RCC_ETHCKSELR, 4, 4, 0, NULL)), /* RTC clock */ - DIV(NO_ID, "ck_hse_rtc", "ck_hse", 0, RCC_RTCDIVR, 0, 7, - CLK_DIVIDER_ALLOW_ZERO), + DIV(NO_ID, "ck_hse_rtc", "ck_hse", 0, RCC_RTCDIVR, 0, 6, 0), COMPOSITE(RTC, "ck_rtc", rtc_src, CLK_OPS_PARENT_ENABLE | CLK_SET_RATE_PARENT, diff --git a/drivers/clk/clk-twl6040.c b/drivers/clk/clk-twl6040.c index ea846f77750b8..0cad5748bf0ec 100644 --- a/drivers/clk/clk-twl6040.c +++ b/drivers/clk/clk-twl6040.c @@ -41,6 +41,43 @@ static int twl6040_pdmclk_is_prepared(struct clk_hw *hw) return pdmclk->enabled; } +static int twl6040_pdmclk_reset_one_clock(struct twl6040_pdmclk *pdmclk, + unsigned int reg) +{ + const u8 reset_mask = TWL6040_HPLLRST; /* Same for HPPLL and LPPLL */ + int ret; + + ret = twl6040_set_bits(pdmclk->twl6040, reg, reset_mask); + if (ret < 0) + return ret; + + ret = twl6040_clear_bits(pdmclk->twl6040, reg, reset_mask); + if (ret < 0) + return ret; + + return 0; +} + +/* + * TWL6040A2 Phoenix Audio IC erratum #6: "PDM Clock Generation Issue At + * Cold Temperature". This affects cold boot and deeper idle states it + * seems. The workaround consists of resetting HPPLL and LPPLL. + */ +static int twl6040_pdmclk_quirk_reset_clocks(struct twl6040_pdmclk *pdmclk) +{ + int ret; + + ret = twl6040_pdmclk_reset_one_clock(pdmclk, TWL6040_REG_HPPLLCTL); + if (ret) + return ret; + + ret = twl6040_pdmclk_reset_one_clock(pdmclk, TWL6040_REG_LPPLLCTL); + if (ret) + return ret; + + return 0; +} + static int twl6040_pdmclk_prepare(struct clk_hw *hw) { struct twl6040_pdmclk *pdmclk = container_of(hw, struct twl6040_pdmclk, @@ -48,8 +85,20 @@ static int twl6040_pdmclk_prepare(struct clk_hw *hw) int ret; ret = twl6040_power(pdmclk->twl6040, 1); - if (!ret) - pdmclk->enabled = 1; + if (ret) + return ret; + + ret = twl6040_pdmclk_quirk_reset_clocks(pdmclk); + if (ret) + goto out_err; + + pdmclk->enabled = 1; + + return 0; + +out_err: + dev_err(pdmclk->dev, "%s: error %i\n", __func__, ret); + twl6040_power(pdmclk->twl6040, 0); return ret; } diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index d2477a5058ac2..96053a96fe2fc 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -57,6 +57,7 @@ struct clk_core { struct clk_core *new_child; unsigned long flags; bool orphan; + bool rpm_enabled; unsigned int enable_count; unsigned int prepare_count; unsigned int protect_count; @@ -81,6 +82,7 @@ struct clk_core { struct clk { struct clk_core *core; + struct device *dev; const char *dev_id; const char *con_id; unsigned long min_rate; @@ -92,9 +94,9 @@ struct clk { /*** runtime pm ***/ static int clk_pm_runtime_get(struct clk_core *core) { - int ret = 0; + int ret; - if (!core->dev) + if (!core->rpm_enabled) return 0; ret = pm_runtime_get_sync(core->dev); @@ -103,7 +105,7 @@ static int clk_pm_runtime_get(struct clk_core *core) static void clk_pm_runtime_put(struct clk_core *core) { - if (!core->dev) + if (!core->rpm_enabled) return; pm_runtime_put_sync(core->dev); @@ -223,7 +225,7 @@ static bool clk_core_is_enabled(struct clk_core *core) * taking enable spinlock, but the below check is needed if one tries * to call it from other places. */ - if (core->dev) { + if (core->rpm_enabled) { pm_runtime_get_noresume(core->dev); if (!pm_runtime_active(core->dev)) { ret = false; @@ -233,7 +235,7 @@ static bool clk_core_is_enabled(struct clk_core *core) ret = core->ops->is_enabled(core->hw); done: - if (core->dev) + if (core->rpm_enabled) pm_runtime_put(core->dev); return ret; @@ -394,16 +396,19 @@ bool clk_hw_is_prepared(const struct clk_hw *hw) { return clk_core_is_prepared(hw->core); } +EXPORT_SYMBOL_GPL(clk_hw_is_prepared); bool clk_hw_rate_is_protected(const struct clk_hw *hw) { return clk_core_rate_is_protected(hw->core); } +EXPORT_SYMBOL_GPL(clk_hw_rate_is_protected); bool clk_hw_is_enabled(const struct clk_hw *hw) { return clk_core_is_enabled(hw->core); } +EXPORT_SYMBOL_GPL(clk_hw_is_enabled); bool __clk_is_enabled(struct clk *clk) { @@ -3209,43 +3214,106 @@ unlock: return ret; } -struct clk *__clk_create_clk(struct clk_hw *hw, const char *dev_id, +/** + * clk_core_link_consumer - Add a clk consumer to the list of consumers in a clk_core + * @core: clk to add consumer to + * @clk: consumer to link to a clk + */ +static void clk_core_link_consumer(struct clk_core *core, struct clk *clk) +{ + clk_prepare_lock(); + hlist_add_head(&clk->clks_node, &core->clks); + clk_prepare_unlock(); +} + +/** + * clk_core_unlink_consumer - Remove a clk consumer from the list of consumers in a clk_core + * @clk: consumer to unlink + */ +static void clk_core_unlink_consumer(struct clk *clk) +{ + lockdep_assert_held(&prepare_lock); + hlist_del(&clk->clks_node); +} + +/** + * alloc_clk - Allocate a clk consumer, but leave it unlinked to the clk_core + * @core: clk to allocate a consumer for + * @dev_id: string describing device name + * @con_id: connection ID string on device + * + * Returns: clk consumer left unlinked from the consumer list + */ +static struct clk *alloc_clk(struct clk_core *core, const char *dev_id, const char *con_id) { struct clk *clk; - /* This is to allow this function to be chained to others */ - if (IS_ERR_OR_NULL(hw)) - return ERR_CAST(hw); - clk = kzalloc(sizeof(*clk), GFP_KERNEL); if (!clk) return ERR_PTR(-ENOMEM); - clk->core = hw->core; + clk->core = core; clk->dev_id = dev_id; clk->con_id = kstrdup_const(con_id, GFP_KERNEL); clk->max_rate = ULONG_MAX; - clk_prepare_lock(); - hlist_add_head(&clk->clks_node, &hw->core->clks); - clk_prepare_unlock(); - return clk; } -/* keep in sync with __clk_put */ -void __clk_free_clk(struct clk *clk) +/** + * free_clk - Free a clk consumer + * @clk: clk consumer to free + * + * Note, this assumes the clk has been unlinked from the clk_core consumer + * list. + */ +static void free_clk(struct clk *clk) { - clk_prepare_lock(); - hlist_del(&clk->clks_node); - clk_prepare_unlock(); - kfree_const(clk->con_id); kfree(clk); } /** + * clk_hw_create_clk: Allocate and link a clk consumer to a clk_core given + * a clk_hw + * @dev: clk consumer device + * @hw: clk_hw associated with the clk being consumed + * @dev_id: string describing device name + * @con_id: connection ID string on device + * + * This is the main function used to create a clk pointer for use by clk + * consumers. It connects a consumer to the clk_core and clk_hw structures + * used by the framework and clk provider respectively. + */ +struct clk *clk_hw_create_clk(struct device *dev, struct clk_hw *hw, + const char *dev_id, const char *con_id) +{ + struct clk *clk; + struct clk_core *core; + + /* This is to allow this function to be chained to others */ + if (IS_ERR_OR_NULL(hw)) + return ERR_CAST(hw); + + core = hw->core; + clk = alloc_clk(core, dev_id, con_id); + if (IS_ERR(clk)) + return clk; + clk->dev = dev; + + if (!try_module_get(core->owner)) { + free_clk(clk); + return ERR_PTR(-ENOENT); + } + + kref_get(&core->ref); + clk_core_link_consumer(core, clk); + + return clk; +} + +/** * clk_register - allocate a new clock, register it and return an opaque cookie * @dev: device that is registering this clock * @hw: link to hardware-specific clock data @@ -3280,7 +3348,8 @@ struct clk *clk_register(struct device *dev, struct clk_hw *hw) core->ops = hw->init->ops; if (dev && pm_runtime_enabled(dev)) - core->dev = dev; + core->rpm_enabled = true; + core->dev = dev; if (dev && dev->driver) core->owner = dev->driver->owner; core->hw = hw; @@ -3320,17 +3389,27 @@ struct clk *clk_register(struct device *dev, struct clk_hw *hw) INIT_HLIST_HEAD(&core->clks); - hw->clk = __clk_create_clk(hw, NULL, NULL); + /* + * Don't call clk_hw_create_clk() here because that would pin the + * provider module to itself and prevent it from ever being removed. + */ + hw->clk = alloc_clk(core, NULL, NULL); if (IS_ERR(hw->clk)) { ret = PTR_ERR(hw->clk); goto fail_parents; } + clk_core_link_consumer(hw->core, hw->clk); + ret = __clk_core_init(core); if (!ret) return hw->clk; - __clk_free_clk(hw->clk); + clk_prepare_lock(); + clk_core_unlink_consumer(hw->clk); + clk_prepare_unlock(); + + free_clk(hw->clk); hw->clk = NULL; fail_parents: @@ -3601,20 +3680,7 @@ EXPORT_SYMBOL_GPL(devm_clk_hw_unregister); /* * clkdev helpers */ -int __clk_get(struct clk *clk) -{ - struct clk_core *core = !clk ? NULL : clk->core; - - if (core) { - if (!try_module_get(core->owner)) - return 0; - kref_get(&core->ref); - } - return 1; -} - -/* keep in sync with __clk_free_clk */ void __clk_put(struct clk *clk) { struct module *owner; @@ -3648,8 +3714,7 @@ void __clk_put(struct clk *clk) module_put(owner); - kfree_const(clk->con_id); - kfree(clk); + free_clk(clk); } /*** clk rate change notifiers ***/ @@ -4006,6 +4071,49 @@ void devm_of_clk_del_provider(struct device *dev) } EXPORT_SYMBOL(devm_of_clk_del_provider); +/* + * Beware the return values when np is valid, but no clock provider is found. + * If name == NULL, the function returns -ENOENT. + * If name != NULL, the function returns -EINVAL. This is because + * of_parse_phandle_with_args() is called even if of_property_match_string() + * returns an error. + */ +static int of_parse_clkspec(const struct device_node *np, int index, + const char *name, struct of_phandle_args *out_args) +{ + int ret = -ENOENT; + + /* Walk up the tree of devices looking for a clock property that matches */ + while (np) { + /* + * For named clocks, first look up the name in the + * "clock-names" property. If it cannot be found, then index + * will be an error code and of_parse_phandle_with_args() will + * return -EINVAL. + */ + if (name) + index = of_property_match_string(np, "clock-names", name); + ret = of_parse_phandle_with_args(np, "clocks", "#clock-cells", + index, out_args); + if (!ret) + break; + if (name && index >= 0) + break; + + /* + * No matching clock found on this node. If the parent node + * has a "clock-ranges" property, then we can try one of its + * clocks. + */ + np = np->parent; + if (np && !of_get_property(np, "clock-ranges", NULL)) + break; + index = 0; + } + + return ret; +} + static struct clk_hw * __of_clk_get_hw_from_provider(struct of_clk_provider *provider, struct of_phandle_args *clkspec) @@ -4021,36 +4129,26 @@ __of_clk_get_hw_from_provider(struct of_clk_provider *provider, return __clk_get_hw(clk); } -struct clk *__of_clk_get_from_provider(struct of_phandle_args *clkspec, - const char *dev_id, const char *con_id) +static struct clk_hw * +of_clk_get_hw_from_clkspec(struct of_phandle_args *clkspec) { struct of_clk_provider *provider; - struct clk *clk = ERR_PTR(-EPROBE_DEFER); - struct clk_hw *hw; + struct clk_hw *hw = ERR_PTR(-EPROBE_DEFER); if (!clkspec) return ERR_PTR(-EINVAL); - /* Check if we have such a provider in our array */ mutex_lock(&of_clk_mutex); list_for_each_entry(provider, &of_clk_providers, link) { if (provider->node == clkspec->np) { hw = __of_clk_get_hw_from_provider(provider, clkspec); - clk = __clk_create_clk(hw, dev_id, con_id); - } - - if (!IS_ERR(clk)) { - if (!__clk_get(clk)) { - __clk_free_clk(clk); - clk = ERR_PTR(-ENOENT); - } - - break; + if (!IS_ERR(hw)) + break; } } mutex_unlock(&of_clk_mutex); - return clk; + return hw; } /** @@ -4063,10 +4161,62 @@ struct clk *__of_clk_get_from_provider(struct of_phandle_args *clkspec, */ struct clk *of_clk_get_from_provider(struct of_phandle_args *clkspec) { - return __of_clk_get_from_provider(clkspec, NULL, __func__); + struct clk_hw *hw = of_clk_get_hw_from_clkspec(clkspec); + + return clk_hw_create_clk(NULL, hw, NULL, __func__); } EXPORT_SYMBOL_GPL(of_clk_get_from_provider); +struct clk_hw *of_clk_get_hw(struct device_node *np, int index, + const char *con_id) +{ + int ret; + struct clk_hw *hw; + struct of_phandle_args clkspec; + + ret = of_parse_clkspec(np, index, con_id, &clkspec); + if (ret) + return ERR_PTR(ret); + + hw = of_clk_get_hw_from_clkspec(&clkspec); + of_node_put(clkspec.np); + + return hw; +} + +static struct clk *__of_clk_get(struct device_node *np, + int index, const char *dev_id, + const char *con_id) +{ + struct clk_hw *hw = of_clk_get_hw(np, index, con_id); + + return clk_hw_create_clk(NULL, hw, dev_id, con_id); +} + +struct clk *of_clk_get(struct device_node *np, int index) +{ + return __of_clk_get(np, index, np->full_name, NULL); +} +EXPORT_SYMBOL(of_clk_get); + +/** + * of_clk_get_by_name() - Parse and lookup a clock referenced by a device node + * @np: pointer to clock consumer node + * @name: name of consumer's clock input, or NULL for the first clock reference + * + * This function parses the clocks and clock-names properties, + * and uses them to look up the struct clk from the registered list of clock + * providers. + */ +struct clk *of_clk_get_by_name(struct device_node *np, const char *name) +{ + if (!np) + return ERR_PTR(-ENOENT); + + return __of_clk_get(np, 0, np->full_name, name); +} +EXPORT_SYMBOL(of_clk_get_by_name); + /** * of_clk_get_parent_count() - Count the number of clocks a device node has * @np: device node to count diff --git a/drivers/clk/clk.h b/drivers/clk/clk.h index b02f5e604e69c..553f531cc232e 100644 --- a/drivers/clk/clk.h +++ b/drivers/clk/clk.h @@ -5,31 +5,36 @@ */ struct clk_hw; +struct device; +struct of_phandle_args; #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK) -struct clk *__of_clk_get_from_provider(struct of_phandle_args *clkspec, - const char *dev_id, const char *con_id); +struct clk_hw *of_clk_get_hw(struct device_node *np, + int index, const char *con_id); +#else /* !CONFIG_COMMON_CLK || !CONFIG_OF */ +static inline struct clk_hw *of_clk_get_hw(struct device_node *np, + int index, const char *con_id) +{ + return ERR_PTR(-ENOENT); +} #endif #ifdef CONFIG_COMMON_CLK -struct clk *__clk_create_clk(struct clk_hw *hw, const char *dev_id, - const char *con_id); -void __clk_free_clk(struct clk *clk); -int __clk_get(struct clk *clk); +struct clk *clk_hw_create_clk(struct device *dev, struct clk_hw *hw, + const char *dev_id, const char *con_id); void __clk_put(struct clk *clk); #else /* All these casts to avoid ifdefs in clkdev... */ static inline struct clk * -__clk_create_clk(struct clk_hw *hw, const char *dev_id, const char *con_id) +clk_hw_create_clk(struct device *dev, struct clk_hw *hw, const char *dev_id, + const char *con_id) { return (struct clk *)hw; } -static inline void __clk_free_clk(struct clk *clk) { } static struct clk_hw *__clk_get_hw(struct clk *clk) { return (struct clk_hw *)clk; } -static inline int __clk_get(struct clk *clk) { return 1; } static inline void __clk_put(struct clk *clk) { } #endif diff --git a/drivers/clk/clkdev.c b/drivers/clk/clkdev.c index 9ab3db8b39883..8c4435c53f09c 100644 --- a/drivers/clk/clkdev.c +++ b/drivers/clk/clkdev.c @@ -27,99 +27,6 @@ static LIST_HEAD(clocks); static DEFINE_MUTEX(clocks_mutex); -#if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK) -static struct clk *__of_clk_get(struct device_node *np, int index, - const char *dev_id, const char *con_id) -{ - struct of_phandle_args clkspec; - struct clk *clk; - int rc; - - rc = of_parse_phandle_with_args(np, "clocks", "#clock-cells", index, - &clkspec); - if (rc) - return ERR_PTR(rc); - - clk = __of_clk_get_from_provider(&clkspec, dev_id, con_id); - of_node_put(clkspec.np); - - return clk; -} - -struct clk *of_clk_get(struct device_node *np, int index) -{ - return __of_clk_get(np, index, np->full_name, NULL); -} -EXPORT_SYMBOL(of_clk_get); - -static struct clk *__of_clk_get_by_name(struct device_node *np, - const char *dev_id, - const char *name) -{ - struct clk *clk = ERR_PTR(-ENOENT); - - /* Walk up the tree of devices looking for a clock that matches */ - while (np) { - int index = 0; - - /* - * For named clocks, first look up the name in the - * "clock-names" property. If it cannot be found, then - * index will be an error code, and of_clk_get() will fail. - */ - if (name) - index = of_property_match_string(np, "clock-names", name); - clk = __of_clk_get(np, index, dev_id, name); - if (!IS_ERR(clk)) { - break; - } else if (name && index >= 0) { - if (PTR_ERR(clk) != -EPROBE_DEFER) - pr_err("ERROR: could not get clock %pOF:%s(%i)\n", - np, name ? name : "", index); - return clk; - } - - /* - * No matching clock found on this node. If the parent node - * has a "clock-ranges" property, then we can try one of its - * clocks. - */ - np = np->parent; - if (np && !of_get_property(np, "clock-ranges", NULL)) - break; - } - - return clk; -} - -/** - * of_clk_get_by_name() - Parse and lookup a clock referenced by a device node - * @np: pointer to clock consumer node - * @name: name of consumer's clock input, or NULL for the first clock reference - * - * This function parses the clocks and clock-names properties, - * and uses them to look up the struct clk from the registered list of clock - * providers. - */ -struct clk *of_clk_get_by_name(struct device_node *np, const char *name) -{ - if (!np) - return ERR_PTR(-ENOENT); - - return __of_clk_get_by_name(np, np->full_name, name); -} -EXPORT_SYMBOL(of_clk_get_by_name); - -#else /* defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK) */ - -static struct clk *__of_clk_get_by_name(struct device_node *np, - const char *dev_id, - const char *name) -{ - return ERR_PTR(-ENOENT); -} -#endif - /* * Find the correct struct clk for the device and connection ID. * We do slightly fuzzy matching here: @@ -163,7 +70,8 @@ static struct clk_lookup *clk_find(const char *dev_id, const char *con_id) return cl; } -struct clk *clk_get_sys(const char *dev_id, const char *con_id) +static struct clk *__clk_get_sys(struct device *dev, const char *dev_id, + const char *con_id) { struct clk_lookup *cl; struct clk *clk = NULL; @@ -174,35 +82,33 @@ struct clk *clk_get_sys(const char *dev_id, const char *con_id) if (!cl) goto out; - clk = __clk_create_clk(cl->clk_hw, dev_id, con_id); + clk = clk_hw_create_clk(dev, cl->clk_hw, dev_id, con_id); if (IS_ERR(clk)) - goto out; - - if (!__clk_get(clk)) { - __clk_free_clk(clk); cl = NULL; - goto out; - } - out: mutex_unlock(&clocks_mutex); return cl ? clk : ERR_PTR(-ENOENT); } + +struct clk *clk_get_sys(const char *dev_id, const char *con_id) +{ + return __clk_get_sys(NULL, dev_id, con_id); +} EXPORT_SYMBOL(clk_get_sys); struct clk *clk_get(struct device *dev, const char *con_id) { const char *dev_id = dev ? dev_name(dev) : NULL; - struct clk *clk; + struct clk_hw *hw; if (dev && dev->of_node) { - clk = __of_clk_get_by_name(dev->of_node, dev_id, con_id); - if (!IS_ERR(clk) || PTR_ERR(clk) == -EPROBE_DEFER) - return clk; + hw = of_clk_get_hw(dev->of_node, 0, con_id); + if (!IS_ERR(hw) || PTR_ERR(hw) == -EPROBE_DEFER) + return clk_hw_create_clk(dev, hw, dev_id, con_id); } - return clk_get_sys(dev_id, con_id); + return __clk_get_sys(dev, dev_id, con_id); } EXPORT_SYMBOL(clk_get); @@ -401,6 +307,23 @@ static struct clk_lookup *__clk_register_clkdev(struct clk_hw *hw, return cl; } +static int do_clk_register_clkdev(struct clk_hw *hw, + struct clk_lookup **cl, const char *con_id, const char *dev_id) +{ + if (IS_ERR(hw)) + return PTR_ERR(hw); + /* + * Since dev_id can be NULL, and NULL is handled specially, we must + * pass it as either a NULL format string, or with "%s". + */ + if (dev_id) + *cl = __clk_register_clkdev(hw, con_id, "%s", dev_id); + else + *cl = __clk_register_clkdev(hw, con_id, NULL); + + return *cl ? 0 : -ENOMEM; +} + /** * clk_register_clkdev - register one clock lookup for a struct clk * @clk: struct clk to associate with all clk_lookups @@ -423,17 +346,8 @@ int clk_register_clkdev(struct clk *clk, const char *con_id, if (IS_ERR(clk)) return PTR_ERR(clk); - /* - * Since dev_id can be NULL, and NULL is handled specially, we must - * pass it as either a NULL format string, or with "%s". - */ - if (dev_id) - cl = __clk_register_clkdev(__clk_get_hw(clk), con_id, "%s", - dev_id); - else - cl = __clk_register_clkdev(__clk_get_hw(clk), con_id, NULL); - - return cl ? 0 : -ENOMEM; + return do_clk_register_clkdev(__clk_get_hw(clk), &cl, con_id, + dev_id); } EXPORT_SYMBOL(clk_register_clkdev); @@ -456,18 +370,75 @@ int clk_hw_register_clkdev(struct clk_hw *hw, const char *con_id, { struct clk_lookup *cl; - if (IS_ERR(hw)) - return PTR_ERR(hw); + return do_clk_register_clkdev(hw, &cl, con_id, dev_id); +} +EXPORT_SYMBOL(clk_hw_register_clkdev); - /* - * Since dev_id can be NULL, and NULL is handled specially, we must - * pass it as either a NULL format string, or with "%s". - */ - if (dev_id) - cl = __clk_register_clkdev(hw, con_id, "%s", dev_id); - else - cl = __clk_register_clkdev(hw, con_id, NULL); +static void devm_clkdev_release(struct device *dev, void *res) +{ + clkdev_drop(*(struct clk_lookup **)res); +} + +static int devm_clk_match_clkdev(struct device *dev, void *res, void *data) +{ + struct clk_lookup **l = res; - return cl ? 0 : -ENOMEM; + return *l == data; } -EXPORT_SYMBOL(clk_hw_register_clkdev); + +/** + * devm_clk_release_clkdev - Resource managed clkdev lookup release + * @dev: device this lookup is bound + * @con_id: connection ID string on device + * @dev_id: format string describing device name + * + * Drop the clkdev lookup created with devm_clk_hw_register_clkdev. + * Normally this function will not need to be called and the resource + * management code will ensure that the resource is freed. + */ +void devm_clk_release_clkdev(struct device *dev, const char *con_id, + const char *dev_id) +{ + struct clk_lookup *cl; + int rval; + + cl = clk_find(dev_id, con_id); + WARN_ON(!cl); + rval = devres_release(dev, devm_clkdev_release, + devm_clk_match_clkdev, cl); + WARN_ON(rval); +} +EXPORT_SYMBOL(devm_clk_release_clkdev); + +/** + * devm_clk_hw_register_clkdev - managed clk lookup registration for clk_hw + * @dev: device this lookup is bound + * @hw: struct clk_hw to associate with all clk_lookups + * @con_id: connection ID string on device + * @dev_id: format string describing device name + * + * con_id or dev_id may be NULL as a wildcard, just as in the rest of + * clkdev. + * + * To make things easier for mass registration, we detect error clk_hws + * from a previous clk_hw_register_*() call, and return the error code for + * those. This is to permit this function to be called immediately + * after clk_hw_register_*(). + */ +int devm_clk_hw_register_clkdev(struct device *dev, struct clk_hw *hw, + const char *con_id, const char *dev_id) +{ + int rval = -ENOMEM; + struct clk_lookup **cl; + + cl = devres_alloc(devm_clkdev_release, sizeof(*cl), GFP_KERNEL); + if (cl) { + rval = do_clk_register_clkdev(hw, cl, con_id, dev_id); + if (!rval) + devres_add(dev, cl); + else + devres_free(cl); + } + return rval; +} +EXPORT_SYMBOL(devm_clk_hw_register_clkdev); diff --git a/drivers/clk/imx/Kconfig b/drivers/clk/imx/Kconfig index 4aae31a234490..0eaf418482800 100644 --- a/drivers/clk/imx/Kconfig +++ b/drivers/clk/imx/Kconfig @@ -8,6 +8,12 @@ config MXC_CLK_SCU bool depends on IMX_SCU +config CLK_IMX8MM + bool "IMX8MM CCM Clock Driver" + depends on ARCH_MXC && ARM64 + help + Build the driver for i.MX8MM CCM Clock Driver + config CLK_IMX8MQ bool "IMX8MQ CCM Clock Driver" depends on ARCH_MXC && ARM64 diff --git a/drivers/clk/imx/Makefile b/drivers/clk/imx/Makefile index 73119fbfa5479..0d5180fbe9883 100644 --- a/drivers/clk/imx/Makefile +++ b/drivers/clk/imx/Makefile @@ -18,12 +18,14 @@ obj-$(CONFIG_MXC_CLK) += \ clk-pllv2.o \ clk-pllv3.o \ clk-pllv4.o \ - clk-sccg-pll.o + clk-sccg-pll.o \ + clk-pll14xx.o obj-$(CONFIG_MXC_CLK_SCU) += \ clk-scu.o \ clk-lpcg-scu.o +obj-$(CONFIG_CLK_IMX8MM) += clk-imx8mm.o obj-$(CONFIG_CLK_IMX8MQ) += clk-imx8mq.o obj-$(CONFIG_CLK_IMX8QXP) += clk-imx8qxp.o clk-imx8qxp-lpcg.o diff --git a/drivers/clk/imx/clk-composite-8m.c b/drivers/clk/imx/clk-composite-8m.c index 527ade1d69331..574fac1a169f4 100644 --- a/drivers/clk/imx/clk-composite-8m.c +++ b/drivers/clk/imx/clk-composite-8m.c @@ -123,7 +123,7 @@ static const struct clk_ops imx8m_clk_composite_divider_ops = { }; struct clk *imx8m_clk_composite_flags(const char *name, - const char **parent_names, + const char * const *parent_names, int num_parents, void __iomem *reg, unsigned long flags) { diff --git a/drivers/clk/imx/clk-imx51-imx53.c b/drivers/clk/imx/clk-imx51-imx53.c index fc8e782d817bd..e91c826bce707 100644 --- a/drivers/clk/imx/clk-imx51-imx53.c +++ b/drivers/clk/imx/clk-imx51-imx53.c @@ -428,6 +428,7 @@ static void __init mx51_clocks_init(struct device_node *np) clk[IMX5_CLK_ESDHC4_PER_GATE] = imx_clk_gate2("esdhc4_per_gate", "esdhc_d_sel", MXC_CCM_CCGR3, 14); clk[IMX5_CLK_USB_PHY_GATE] = imx_clk_gate2("usb_phy_gate", "usb_phy_sel", MXC_CCM_CCGR2, 0); clk[IMX5_CLK_HSI2C_GATE] = imx_clk_gate2("hsi2c_gate", "ipg", MXC_CCM_CCGR1, 22); + clk[IMX5_CLK_SCC2_IPG_GATE] = imx_clk_gate2("scc2_gate", "ipg", MXC_CCM_CCGR1, 30); clk[IMX5_CLK_MIPI_HSC1_GATE] = imx_clk_gate2_flags("mipi_hsc1_gate", "ipg", MXC_CCM_CCGR4, 6, CLK_IS_CRITICAL); clk[IMX5_CLK_MIPI_HSC2_GATE] = imx_clk_gate2_flags("mipi_hsc2_gate", "ipg", MXC_CCM_CCGR4, 8, CLK_IS_CRITICAL); clk[IMX5_CLK_MIPI_ESC_GATE] = imx_clk_gate2_flags("mipi_esc_gate", "ipg", MXC_CCM_CCGR4, 10, CLK_IS_CRITICAL); diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c index 716eac3136b4d..708e7c5590ddc 100644 --- a/drivers/clk/imx/clk-imx6q.c +++ b/drivers/clk/imx/clk-imx6q.c @@ -471,6 +471,7 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-anatop"); anatop_base = base = of_iomap(np, 0); WARN_ON(!base); + of_node_put(np); /* Audio/video PLL post dividers do not work on i.MX6q revision 1.0 */ if (clk_on_imx6q() && imx_get_soc_revision() == IMX_CHIP_REVISION_1_0) { diff --git a/drivers/clk/imx/clk-imx6sx.c b/drivers/clk/imx/clk-imx6sx.c index 18527a335ace3..91558b09bf9eb 100644 --- a/drivers/clk/imx/clk-imx6sx.c +++ b/drivers/clk/imx/clk-imx6sx.c @@ -151,6 +151,7 @@ static void __init imx6sx_clocks_init(struct device_node *ccm_node) np = of_find_compatible_node(NULL, NULL, "fsl,imx6sx-anatop"); base = of_iomap(np, 0); WARN_ON(!base); + of_node_put(np); clks[IMX6SX_PLL1_BYPASS_SRC] = imx_clk_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); clks[IMX6SX_PLL2_BYPASS_SRC] = imx_clk_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); diff --git a/drivers/clk/imx/clk-imx7d.c b/drivers/clk/imx/clk-imx7d.c index 06c105d580a45..cfbd8d4edb859 100644 --- a/drivers/clk/imx/clk-imx7d.c +++ b/drivers/clk/imx/clk-imx7d.c @@ -404,6 +404,7 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node) np = of_find_compatible_node(NULL, NULL, "fsl,imx7d-anatop"); base = of_iomap(np, 0); WARN_ON(!base); + of_node_put(np); clks[IMX7D_PLL_ARM_MAIN_SRC] = imx_clk_mux("pll_arm_main_src", base + 0x60, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); clks[IMX7D_PLL_DRAM_MAIN_SRC] = imx_clk_mux("pll_dram_main_src", base + 0x70, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); diff --git a/drivers/clk/imx/clk-imx7ulp.c b/drivers/clk/imx/clk-imx7ulp.c index 4e18f629f8236..ce306631e8441 100644 --- a/drivers/clk/imx/clk-imx7ulp.c +++ b/drivers/clk/imx/clk-imx7ulp.c @@ -48,8 +48,8 @@ static void __init imx7ulp_clk_scg1_init(struct device_node *np) struct clk_hw **clks; void __iomem *base; - clk_data = kzalloc(sizeof(*clk_data) + sizeof(*clk_data->hws) * - IMX7ULP_CLK_SCG1_END, GFP_KERNEL); + clk_data = kzalloc(struct_size(clk_data, hws, IMX7ULP_CLK_SCG1_END), + GFP_KERNEL); if (!clk_data) return; @@ -136,8 +136,8 @@ static void __init imx7ulp_clk_pcc2_init(struct device_node *np) struct clk_hw **clks; void __iomem *base; - clk_data = kzalloc(sizeof(*clk_data) + sizeof(*clk_data->hws) * - IMX7ULP_CLK_PCC2_END, GFP_KERNEL); + clk_data = kzalloc(struct_size(clk_data, hws, IMX7ULP_CLK_PCC2_END), + GFP_KERNEL); if (!clk_data) return; @@ -183,8 +183,8 @@ static void __init imx7ulp_clk_pcc3_init(struct device_node *np) struct clk_hw **clks; void __iomem *base; - clk_data = kzalloc(sizeof(*clk_data) + sizeof(*clk_data->hws) * - IMX7ULP_CLK_PCC3_END, GFP_KERNEL); + clk_data = kzalloc(struct_size(clk_data, hws, IMX7ULP_CLK_PCC3_END), + GFP_KERNEL); if (!clk_data) return; @@ -228,8 +228,8 @@ static void __init imx7ulp_clk_smc1_init(struct device_node *np) struct clk_hw **clks; void __iomem *base; - clk_data = kzalloc(sizeof(*clk_data) + sizeof(*clk_data->hws) * - IMX7ULP_CLK_SMC1_END, GFP_KERNEL); + clk_data = kzalloc(struct_size(clk_data, hws, IMX7ULP_CLK_SMC1_END), + GFP_KERNEL); if (!clk_data) return; diff --git a/drivers/clk/imx/clk-imx8mm.c b/drivers/clk/imx/clk-imx8mm.c new file mode 100644 index 0000000000000..1ef8438e3d6d4 --- /dev/null +++ b/drivers/clk/imx/clk-imx8mm.c @@ -0,0 +1,675 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2017-2018 NXP. + */ + +#include <dt-bindings/clock/imx8mm-clock.h> +#include <linux/clk.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/platform_device.h> +#include <linux/types.h> + +#include "clk.h" + +static u32 share_count_sai1; +static u32 share_count_sai2; +static u32 share_count_sai3; +static u32 share_count_sai4; +static u32 share_count_sai5; +static u32 share_count_sai6; +static u32 share_count_dcss; +static u32 share_count_pdm; +static u32 share_count_nand; + +#define PLL_1416X_RATE(_rate, _m, _p, _s) \ + { \ + .rate = (_rate), \ + .mdiv = (_m), \ + .pdiv = (_p), \ + .sdiv = (_s), \ + } + +#define PLL_1443X_RATE(_rate, _m, _p, _s, _k) \ + { \ + .rate = (_rate), \ + .mdiv = (_m), \ + .pdiv = (_p), \ + .sdiv = (_s), \ + .kdiv = (_k), \ + } + +static const struct imx_pll14xx_rate_table imx8mm_pll1416x_tbl[] = { + PLL_1416X_RATE(1800000000U, 225, 3, 0), + PLL_1416X_RATE(1600000000U, 200, 3, 0), + PLL_1416X_RATE(1200000000U, 300, 3, 1), + PLL_1416X_RATE(1000000000U, 250, 3, 1), + PLL_1416X_RATE(800000000U, 200, 3, 1), + PLL_1416X_RATE(750000000U, 250, 2, 2), + PLL_1416X_RATE(700000000U, 350, 3, 2), + PLL_1416X_RATE(600000000U, 300, 3, 2), +}; + +static const struct imx_pll14xx_rate_table imx8mm_audiopll_tbl[] = { + PLL_1443X_RATE(786432000U, 655, 5, 2, 23593), + PLL_1443X_RATE(722534400U, 301, 5, 1, 3670), +}; + +static const struct imx_pll14xx_rate_table imx8mm_videopll_tbl[] = { + PLL_1443X_RATE(650000000U, 325, 3, 2, 0), + PLL_1443X_RATE(594000000U, 198, 2, 2, 0), +}; + +static const struct imx_pll14xx_rate_table imx8mm_drampll_tbl[] = { + PLL_1443X_RATE(650000000U, 325, 3, 2, 0), +}; + +static struct imx_pll14xx_clk imx8mm_audio_pll __initdata = { + .type = PLL_1443X, + .rate_table = imx8mm_audiopll_tbl, + .rate_count = ARRAY_SIZE(imx8mm_audiopll_tbl), +}; + +static struct imx_pll14xx_clk imx8mm_video_pll __initdata = { + .type = PLL_1443X, + .rate_table = imx8mm_videopll_tbl, + .rate_count = ARRAY_SIZE(imx8mm_videopll_tbl), +}; + +static struct imx_pll14xx_clk imx8mm_dram_pll __initdata = { + .type = PLL_1443X, + .rate_table = imx8mm_drampll_tbl, + .rate_count = ARRAY_SIZE(imx8mm_drampll_tbl), +}; + +static struct imx_pll14xx_clk imx8mm_arm_pll __initdata = { + .type = PLL_1416X, + .rate_table = imx8mm_pll1416x_tbl, + .rate_count = ARRAY_SIZE(imx8mm_pll1416x_tbl), +}; + +static struct imx_pll14xx_clk imx8mm_gpu_pll __initdata = { + .type = PLL_1416X, + .rate_table = imx8mm_pll1416x_tbl, + .rate_count = ARRAY_SIZE(imx8mm_pll1416x_tbl), +}; + +static struct imx_pll14xx_clk imx8mm_vpu_pll __initdata = { + .type = PLL_1416X, + .rate_table = imx8mm_pll1416x_tbl, + .rate_count = ARRAY_SIZE(imx8mm_pll1416x_tbl), +}; + +static struct imx_pll14xx_clk imx8mm_sys_pll __initdata = { + .type = PLL_1416X, + .rate_table = imx8mm_pll1416x_tbl, + .rate_count = ARRAY_SIZE(imx8mm_pll1416x_tbl), +}; + +static const char *pll_ref_sels[] = { "osc_24m", "dummy", "dummy", "dummy", }; +static const char *audio_pll1_bypass_sels[] = {"audio_pll1", "audio_pll1_ref_sel", }; +static const char *audio_pll2_bypass_sels[] = {"audio_pll2", "audio_pll2_ref_sel", }; +static const char *video_pll1_bypass_sels[] = {"video_pll1", "video_pll1_ref_sel", }; +static const char *dram_pll_bypass_sels[] = {"dram_pll", "dram_pll_ref_sel", }; +static const char *gpu_pll_bypass_sels[] = {"gpu_pll", "gpu_pll_ref_sel", }; +static const char *vpu_pll_bypass_sels[] = {"vpu_pll", "vpu_pll_ref_sel", }; +static const char *arm_pll_bypass_sels[] = {"arm_pll", "arm_pll_ref_sel", }; +static const char *sys_pll1_bypass_sels[] = {"sys_pll1", "sys_pll1_ref_sel", }; +static const char *sys_pll2_bypass_sels[] = {"sys_pll2", "sys_pll2_ref_sel", }; +static const char *sys_pll3_bypass_sels[] = {"sys_pll3", "sys_pll3_ref_sel", }; + +/* CCM ROOT */ +static const char *imx8mm_a53_sels[] = {"osc_24m", "arm_pll_out", "sys_pll2_500m", "sys_pll2_1000m", + "sys_pll1_800m", "sys_pll1_400m", "audio_pll1_out", "sys_pll3_out", }; + +static const char *imx8mm_m4_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll2_250m", "sys_pll1_266m", + "sys_pll1_800m", "audio_pll1_out", "video_pll1_out", "sys_pll3_out", }; + +static const char *imx8mm_vpu_sels[] = {"osc_24m", "arm_pll_out", "sys_pll2_500m", "sys_pll2_1000m", + "sys_pll1_800m", "sys_pll1_400m", "audio_pll1_out", "vpu_pll_out", }; + +static const char *imx8mm_gpu3d_sels[] = {"osc_24m", "gpu_pll_out", "sys_pll1_800m", "sys_pll3_out", + "sys_pll2_1000m", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", }; + +static const char *imx8mm_gpu2d_sels[] = {"osc_24m", "gpu_pll_out", "sys_pll1_800m", "sys_pll3_out", + "sys_pll2_1000m", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", }; + +static const char *imx8mm_main_axi_sels[] = {"osc_24m", "sys_pll2_333m", "sys_pll1_800m", "sys_pll2_250m", + "sys_pll2_1000m", "audio_pll1_out", "video_pll1_out", "sys_pll1_100m",}; + +static const char *imx8mm_enet_axi_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll1_800m", "sys_pll2_250m", + "sys_pll2_200m", "audio_pll1_out", "video_pll1_out", "sys_pll3_out", }; + +static const char *imx8mm_nand_usdhc_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll1_800m", "sys_pll2_200m", + "sys_pll1_133m", "sys_pll3_out", "sys_pll2_250m", "audio_pll1_out", }; + +static const char *imx8mm_vpu_bus_sels[] = {"osc_24m", "sys_pll1_800m", "vpu_pll_out", "audio_pll2_out", + "sys_pll3_out", "sys_pll2_1000m", "sys_pll2_200m", "sys_pll1_100m", }; + +static const char *imx8mm_disp_axi_sels[] = {"osc_24m", "sys_pll2_1000m", "sys_pll1_800m", "sys_pll3_out", + "sys_pll1_40m", "audio_pll2_out", "clk_ext1", "clk_ext4", }; + +static const char *imx8mm_disp_apb_sels[] = {"osc_24m", "sys_pll2_125m", "sys_pll1_800m", "sys_pll3_out", + "sys_pll1_40m", "audio_pll2_out", "clk_ext1", "clk_ext3", }; + +static const char *imx8mm_disp_rtrm_sels[] = {"osc_24m", "sys_pll1_800m", "sys_pll2_200m", "sys_pll2_1000m", + "audio_pll1_out", "video_pll1_out", "clk_ext2", "clk_ext3", }; + +static const char *imx8mm_usb_bus_sels[] = {"osc_24m", "sys_pll2_500m", "sys_pll1_800m", "sys_pll2_100m", + "sys_pll2_200m", "clk_ext2", "clk_ext4", "audio_pll2_out", }; + +static const char *imx8mm_gpu_axi_sels[] = {"osc_24m", "sys_pll1_800m", "gpu_pll_out", "sys_pll3_out", "sys_pll2_1000m", + "audio_pll1_out", "video_pll1_out", "audio_pll2_out", }; + +static const char *imx8mm_gpu_ahb_sels[] = {"osc_24m", "sys_pll1_800m", "gpu_pll_out", "sys_pll3_out", "sys_pll2_1000m", + "audio_pll1_out", "video_pll1_out", "audio_pll2_out", }; + +static const char *imx8mm_noc_sels[] = {"osc_24m", "sys_pll1_800m", "sys_pll3_out", "sys_pll2_1000m", "sys_pll2_500m", + "audio_pll1_out", "video_pll1_out", "audio_pll2_out", }; + +static const char *imx8mm_noc_apb_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll3_out", "sys_pll2_333m", "sys_pll2_200m", + "sys_pll1_800m", "audio_pll1_out", "video_pll1_out", }; + +static const char *imx8mm_ahb_sels[] = {"osc_24m", "sys_pll1_133m", "sys_pll1_800m", "sys_pll1_400m", + "sys_pll2_125m", "sys_pll3_out", "audio_pll1_out", "video_pll1_out", }; + +static const char *imx8mm_audio_ahb_sels[] = {"osc_24m", "sys_pll2_500m", "sys_pll1_800m", "sys_pll2_1000m", + "sys_pll2_166m", "sys_pll3_out", "audio_pll1_out", "video_pll1_out", }; + +static const char *imx8mm_dram_alt_sels[] = {"osc_24m", "sys_pll1_800m", "sys_pll1_100m", "sys_pll2_500m", + "sys_pll2_1000m", "sys_pll3_out", "audio_pll1_out", "sys_pll1_266m", }; + +static const char *imx8mm_dram_apb_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", "sys_pll1_160m", + "sys_pll1_800m", "sys_pll3_out", "sys_pll2_250m", "audio_pll2_out", }; + +static const char *imx8mm_vpu_g1_sels[] = {"osc_24m", "vpu_pll_out", "sys_pll1_800m", "sys_pll2_1000m", + "sys_pll1_100m", "sys_pll2_125m", "sys_pll3_out", "audio_pll1_out", }; + +static const char *imx8mm_vpu_g2_sels[] = {"osc_24m", "vpu_pll_out", "sys_pll1_800m", "sys_pll2_1000m", + "sys_pll1_100m", "sys_pll2_125m", "sys_pll3_out", "audio_pll1_out", }; + +static const char *imx8mm_disp_dtrc_sels[] = {"osc_24m", "video_pll2_out", "sys_pll1_800m", "sys_pll2_1000m", + "sys_pll1_160m", "video_pll1_out", "sys_pll3_out", "audio_pll2_out", }; + +static const char *imx8mm_disp_dc8000_sels[] = {"osc_24m", "video_pll2_out", "sys_pll1_800m", "sys_pll2_1000m", + "sys_pll1_160m", "video_pll1_out", "sys_pll3_out", "audio_pll2_out", }; + +static const char *imx8mm_pcie1_ctrl_sels[] = {"osc_24m", "sys_pll2_250m", "sys_pll2_200m", "sys_pll1_266m", + "sys_pll1_800m", "sys_pll2_500m", "sys_pll2_333m", "sys_pll3_out", }; + +static const char *imx8mm_pcie1_phy_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll2_500m", "clk_ext1", "clk_ext2", + "clk_ext3", "clk_ext4", "sys_pll1_400m", }; + +static const char *imx8mm_pcie1_aux_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll2_50m", "sys_pll3_out", + "sys_pll2_100m", "sys_pll1_80m", "sys_pll1_160m", "sys_pll1_200m", }; + +static const char *imx8mm_dc_pixel_sels[] = {"osc_24m", "video_pll1_out", "audio_pll2_out", "audio_pll1_out", + "sys_pll1_800m", "sys_pll2_1000m", "sys_pll3_out", "clk_ext4", }; + +static const char *imx8mm_lcdif_pixel_sels[] = {"osc_24m", "video_pll1_out", "audio_pll2_out", "audio_pll1_out", + "sys_pll1_800m", "sys_pll2_1000m", "sys_pll3_out", "clk_ext4", }; + +static const char *imx8mm_sai1_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "sys_pll1_133m", "osc_hdmi", "clk_ext1", "clk_ext2", }; + +static const char *imx8mm_sai2_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "sys_pll1_133m", "osc_hdmi", "clk_ext2", "clk_ext3", }; + +static const char *imx8mm_sai3_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "sys_pll1_133m", "osc_hdmi", "clk_ext3", "clk_ext4", }; + +static const char *imx8mm_sai4_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "sys_pll1_133m", "osc_hdmi", "clk_ext1", "clk_ext2", }; + +static const char *imx8mm_sai5_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "sys_pll1_133m", "osc_hdmi", "clk_ext2", "clk_ext3", }; + +static const char *imx8mm_sai6_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "sys_pll1_133m", "osc_hdmi", "clk_ext3", "clk_ext4", }; + +static const char *imx8mm_spdif1_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "sys_pll1_133m", "osc_hdmi", "clk_ext2", "clk_ext3", }; + +static const char *imx8mm_spdif2_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "sys_pll1_133m", "osc_hdmi", "clk_ext3", "clk_ext4", }; + +static const char *imx8mm_enet_ref_sels[] = {"osc_24m", "sys_pll2_125m", "sys_pll2_50m", "sys_pll2_100m", + "sys_pll1_160m", "audio_pll1_out", "video_pll1_out", "clk_ext4", }; + +static const char *imx8mm_enet_timer_sels[] = {"osc_24m", "sys_pll2_100m", "audio_pll1_out", "clk_ext1", "clk_ext2", + "clk_ext3", "clk_ext4", "video_pll1_out", }; + +static const char *imx8mm_enet_phy_sels[] = {"osc_24m", "sys_pll2_50m", "sys_pll2_125m", "sys_pll2_200m", + "sys_pll2_500m", "video_pll1_out", "audio_pll2_out", }; + +static const char *imx8mm_nand_sels[] = {"osc_24m", "sys_pll2_500m", "audio_pll1_out", "sys_pll1_400m", + "audio_pll2_out", "sys_pll3_out", "sys_pll2_250m", "video_pll1_out", }; + +static const char *imx8mm_qspi_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m", "sys_pll2_500m", + "audio_pll2_out", "sys_pll1_266m", "sys_pll3_out", "sys_pll1_100m", }; + +static const char *imx8mm_usdhc1_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m", "sys_pll2_500m", + "sys_pll3_out", "sys_pll1_266m", "audio_pll2_out", "sys_pll1_100m", }; + +static const char *imx8mm_usdhc2_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m", "sys_pll2_500m", + "sys_pll3_out", "sys_pll1_266m", "audio_pll2_out", "sys_pll1_100m", }; + +static const char *imx8mm_i2c1_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", "sys_pll3_out", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", "sys_pll1_133m", }; + +static const char *imx8mm_i2c2_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", "sys_pll3_out", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", "sys_pll1_133m", }; + +static const char *imx8mm_i2c3_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", "sys_pll3_out", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", "sys_pll1_133m", }; + +static const char *imx8mm_i2c4_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", "sys_pll3_out", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", "sys_pll1_133m", }; + +static const char *imx8mm_uart1_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m", "sys_pll2_100m", + "sys_pll3_out", "clk_ext2", "clk_ext4", "audio_pll2_out", }; + +static const char *imx8mm_uart2_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m", "sys_pll2_100m", + "sys_pll3_out", "clk_ext2", "clk_ext3", "audio_pll2_out", }; + +static const char *imx8mm_uart3_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m", "sys_pll2_100m", + "sys_pll3_out", "clk_ext2", "clk_ext4", "audio_pll2_out", }; + +static const char *imx8mm_uart4_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m", "sys_pll2_100m", + "sys_pll3_out", "clk_ext2", "clk_ext3", "audio_pll2_out", }; + +static const char *imx8mm_usb_core_sels[] = {"osc_24m", "sys_pll1_100m", "sys_pll1_40m", "sys_pll2_100m", + "sys_pll2_200m", "clk_ext2", "clk_ext3", "audio_pll2_out", }; + +static const char *imx8mm_usb_phy_sels[] = {"osc_24m", "sys_pll1_100m", "sys_pll1_40m", "sys_pll2_100m", + "sys_pll2_200m", "clk_ext2", "clk_ext3", "audio_pll2_out", }; + +static const char *imx8mm_ecspi1_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", "sys_pll1_160m", + "sys_pll1_800m", "sys_pll3_out", "sys_pll2_250m", "audio_pll2_out", }; + +static const char *imx8mm_ecspi2_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", "sys_pll1_160m", + "sys_pll1_800m", "sys_pll3_out", "sys_pll2_250m", "audio_pll2_out", }; + +static const char *imx8mm_pwm1_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m", "sys_pll1_40m", + "sys_pll3_out", "clk_ext1", "sys_pll1_80m", "video_pll1_out", }; + +static const char *imx8mm_pwm2_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m", "sys_pll1_40m", + "sys_pll3_out", "clk_ext1", "sys_pll1_80m", "video_pll1_out", }; + +static const char *imx8mm_pwm3_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m", "sys_pll1_40m", + "sys3_pll2_out", "clk_ext2", "sys_pll1_80m", "video_pll1_out", }; + +static const char *imx8mm_pwm4_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m", "sys_pll1_40m", + "sys_pll3_out", "clk_ext2", "sys_pll1_80m", "video_pll1_out", }; + +static const char *imx8mm_gpt1_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_400m", "sys_pll1_40m", + "video_pll1_out", "sys_pll1_800m", "audio_pll1_out", "clk_ext1" }; + +static const char *imx8mm_wdog_sels[] = {"osc_24m", "sys_pll1_133m", "sys_pll1_160m", "vpu_pll_out", + "sys_pll2_125m", "sys_pll3_out", "sys_pll1_80m", "sys_pll2_166m", }; + +static const char *imx8mm_wrclk_sels[] = {"osc_24m", "sys_pll1_40m", "vpu_pll_out", "sys_pll3_out", "sys_pll2_200m", + "sys_pll1_266m", "sys_pll2_500m", "sys_pll1_100m", }; + +static const char *imx8mm_dsi_core_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_250m", "sys_pll1_800m", + "sys_pll2_1000m", "sys_pll3_out", "audio_pll2_out", "video_pll1_out", }; + +static const char *imx8mm_dsi_phy_sels[] = {"osc_24m", "sys_pll2_125m", "sys_pll2_100m", "sys_pll1_800m", + "sys_pll2_1000m", "clk_ext2", "audio_pll2_out", "video_pll1_out", }; + +static const char *imx8mm_dsi_dbi_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_100m", "sys_pll1_800m", + "sys_pll2_1000m", "sys_pll3_out", "audio_pll2_out", "video_pll1_out", }; + +static const char *imx8mm_usdhc3_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m", "sys_pll2_500m", + "sys_pll3_out", "sys_pll1_266m", "audio_pll2_clk", "sys_pll1_100m", }; + +static const char *imx8mm_csi1_core_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_250m", "sys_pll1_800m", + "sys_pll2_1000m", "sys_pll3_out", "audio_pll2_out", "video_pll1_out", }; + +static const char *imx8mm_csi1_phy_sels[] = {"osc_24m", "sys_pll2_333m", "sys_pll2_100m", "sys_pll1_800m", + "sys_pll2_1000m", "clk_ext2", "audio_pll2_out", "video_pll1_out", }; + +static const char *imx8mm_csi1_esc_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_80m", "sys_pll1_800m", + "sys_pll2_1000m", "sys_pll3_out", "clk_ext3", "audio_pll2_out", }; + +static const char *imx8mm_csi2_core_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_250m", "sys_pll1_800m", + "sys_pll2_1000m", "sys_pll3_out", "audio_pll2_out", "video_pll1_out", }; + +static const char *imx8mm_csi2_phy_sels[] = {"osc_24m", "sys_pll2_333m", "sys_pll2_100m", "sys_pll1_800m", + "sys_pll2_1000m", "clk_ext2", "audio_pll2_out", "video_pll1_out", }; + +static const char *imx8mm_csi2_esc_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_80m", "sys_pll1_800m", + "sys_pll2_1000m", "sys_pll3_out", "clk_ext3", "audio_pll2_out", }; + +static const char *imx8mm_pcie2_ctrl_sels[] = {"osc_24m", "sys_pll2_250m", "sys_pll2_200m", "sys_pll1_266m", + "sys_pll1_800m", "sys_pll2_500m", "sys_pll2_333m", "sys_pll3_out", }; + +static const char *imx8mm_pcie2_phy_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll2_500m", "clk_ext1", + "clk_ext2", "clk_ext3", "clk_ext4", "sys_pll1_400m", }; + +static const char *imx8mm_pcie2_aux_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll2_50m", "sys_pll3_out", + "sys_pll2_100m", "sys_pll1_80m", "sys_pll1_160m", "sys_pll1_200m", }; + +static const char *imx8mm_ecspi3_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", "sys_pll1_160m", + "sys_pll1_800m", "sys_pll3_out", "sys_pll2_250m", "audio_pll2_out", }; + +static const char *imx8mm_pdm_sels[] = {"osc_24m", "sys_pll2_100m", "audio_pll1_out", "sys_pll1_800m", + "sys_pll2_1000m", "sys_pll3_out", "clk_ext3", "audio_pll2_out", }; + +static const char *imx8mm_vpu_h1_sels[] = {"osc_24m", "vpu_pll_out", "sys_pll1_800m", "sys_pll2_1000m", + "audio_pll2_clk", "sys_pll2_125m", "sys_pll3_clk", "audio_pll1_out", }; + +static const char *imx8mm_dram_core_sels[] = {"dram_pll_out", "dram_alt_root", }; + +static const char *imx8mm_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "osc_27m", "sys_pll1_200m", "audio_pll2_clk", + "vpu_pll", "sys_pll1_80m", }; + +static struct clk *clks[IMX8MM_CLK_END]; +static struct clk_onecell_data clk_data; + +static struct clk ** const uart_clks[] __initconst = { + &clks[IMX8MM_CLK_UART1_ROOT], + &clks[IMX8MM_CLK_UART2_ROOT], + &clks[IMX8MM_CLK_UART3_ROOT], + &clks[IMX8MM_CLK_UART4_ROOT], + NULL +}; + +static int __init imx8mm_clocks_init(struct device_node *ccm_node) +{ + struct device_node *np; + void __iomem *base; + int ret; + + clks[IMX8MM_CLK_DUMMY] = imx_clk_fixed("dummy", 0); + clks[IMX8MM_CLK_24M] = of_clk_get_by_name(ccm_node, "osc_24m"); + clks[IMX8MM_CLK_32K] = of_clk_get_by_name(ccm_node, "osc_32k"); + clks[IMX8MM_CLK_EXT1] = of_clk_get_by_name(ccm_node, "clk_ext1"); + clks[IMX8MM_CLK_EXT2] = of_clk_get_by_name(ccm_node, "clk_ext2"); + clks[IMX8MM_CLK_EXT3] = of_clk_get_by_name(ccm_node, "clk_ext3"); + clks[IMX8MM_CLK_EXT4] = of_clk_get_by_name(ccm_node, "clk_ext4"); + + np = of_find_compatible_node(NULL, NULL, "fsl,imx8mm-anatop"); + base = of_iomap(np, 0); + if (WARN_ON(!base)) + return -ENOMEM; + + clks[IMX8MM_AUDIO_PLL1_REF_SEL] = imx_clk_mux("audio_pll1_ref_sel", base + 0x0, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + clks[IMX8MM_AUDIO_PLL2_REF_SEL] = imx_clk_mux("audio_pll2_ref_sel", base + 0x14, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + clks[IMX8MM_VIDEO_PLL1_REF_SEL] = imx_clk_mux("video_pll1_ref_sel", base + 0x28, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + clks[IMX8MM_DRAM_PLL_REF_SEL] = imx_clk_mux("dram_pll_ref_sel", base + 0x50, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + clks[IMX8MM_GPU_PLL_REF_SEL] = imx_clk_mux("gpu_pll_ref_sel", base + 0x64, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + clks[IMX8MM_VPU_PLL_REF_SEL] = imx_clk_mux("vpu_pll_ref_sel", base + 0x74, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + clks[IMX8MM_ARM_PLL_REF_SEL] = imx_clk_mux("arm_pll_ref_sel", base + 0x84, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + clks[IMX8MM_SYS_PLL1_REF_SEL] = imx_clk_mux("sys_pll1_ref_sel", base + 0x94, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + clks[IMX8MM_SYS_PLL2_REF_SEL] = imx_clk_mux("sys_pll2_ref_sel", base + 0x104, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + clks[IMX8MM_SYS_PLL3_REF_SEL] = imx_clk_mux("sys_pll3_ref_sel", base + 0x114, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + + clks[IMX8MM_AUDIO_PLL1] = imx_clk_pll14xx("audio_pll1", "audio_pll1_ref_sel", base, &imx8mm_audio_pll); + clks[IMX8MM_AUDIO_PLL2] = imx_clk_pll14xx("audio_pll2", "audio_pll2_ref_sel", base + 0x14, &imx8mm_audio_pll); + clks[IMX8MM_VIDEO_PLL1] = imx_clk_pll14xx("video_pll1", "video_pll1_ref_sel", base + 0x28, &imx8mm_video_pll); + clks[IMX8MM_DRAM_PLL] = imx_clk_pll14xx("dram_pll", "dram_pll_ref_sel", base + 0x50, &imx8mm_dram_pll); + clks[IMX8MM_GPU_PLL] = imx_clk_pll14xx("gpu_pll", "gpu_pll_ref_sel", base + 0x64, &imx8mm_gpu_pll); + clks[IMX8MM_VPU_PLL] = imx_clk_pll14xx("vpu_pll", "vpu_pll_ref_sel", base + 0x74, &imx8mm_vpu_pll); + clks[IMX8MM_ARM_PLL] = imx_clk_pll14xx("arm_pll", "arm_pll_ref_sel", base + 0x84, &imx8mm_arm_pll); + clks[IMX8MM_SYS_PLL1] = imx_clk_pll14xx("sys_pll1", "sys_pll1_ref_sel", base + 0x94, &imx8mm_sys_pll); + clks[IMX8MM_SYS_PLL2] = imx_clk_pll14xx("sys_pll2", "sys_pll2_ref_sel", base + 0x104, &imx8mm_sys_pll); + clks[IMX8MM_SYS_PLL3] = imx_clk_pll14xx("sys_pll3", "sys_pll3_ref_sel", base + 0x114, &imx8mm_sys_pll); + + /* PLL bypass out */ + clks[IMX8MM_AUDIO_PLL1_BYPASS] = imx_clk_mux_flags("audio_pll1_bypass", base, 4, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels), CLK_SET_RATE_PARENT); + clks[IMX8MM_AUDIO_PLL2_BYPASS] = imx_clk_mux_flags("audio_pll2_bypass", base + 0x14, 4, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels), CLK_SET_RATE_PARENT); + clks[IMX8MM_VIDEO_PLL1_BYPASS] = imx_clk_mux_flags("video_pll1_bypass", base + 0x28, 4, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels), CLK_SET_RATE_PARENT); + clks[IMX8MM_DRAM_PLL_BYPASS] = imx_clk_mux_flags("dram_pll_bypass", base + 0x50, 4, 1, dram_pll_bypass_sels, ARRAY_SIZE(dram_pll_bypass_sels), CLK_SET_RATE_PARENT); + clks[IMX8MM_GPU_PLL_BYPASS] = imx_clk_mux_flags("gpu_pll_bypass", base + 0x64, 4, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels), CLK_SET_RATE_PARENT); + clks[IMX8MM_VPU_PLL_BYPASS] = imx_clk_mux_flags("vpu_pll_bypass", base + 0x74, 4, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels), CLK_SET_RATE_PARENT); + clks[IMX8MM_ARM_PLL_BYPASS] = imx_clk_mux_flags("arm_pll_bypass", base + 0x84, 4, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT); + clks[IMX8MM_SYS_PLL1_BYPASS] = imx_clk_mux_flags("sys_pll1_bypass", base + 0x94, 4, 1, sys_pll1_bypass_sels, ARRAY_SIZE(sys_pll1_bypass_sels), CLK_SET_RATE_PARENT); + clks[IMX8MM_SYS_PLL2_BYPASS] = imx_clk_mux_flags("sys_pll2_bypass", base + 0x104, 4, 1, sys_pll2_bypass_sels, ARRAY_SIZE(sys_pll2_bypass_sels), CLK_SET_RATE_PARENT); + clks[IMX8MM_SYS_PLL3_BYPASS] = imx_clk_mux_flags("sys_pll3_bypass", base + 0x114, 4, 1, sys_pll3_bypass_sels, ARRAY_SIZE(sys_pll3_bypass_sels), CLK_SET_RATE_PARENT); + + /* unbypass all the plls */ + clk_set_parent(clks[IMX8MM_AUDIO_PLL1_BYPASS], clks[IMX8MM_AUDIO_PLL1]); + clk_set_parent(clks[IMX8MM_AUDIO_PLL2_BYPASS], clks[IMX8MM_AUDIO_PLL2]); + clk_set_parent(clks[IMX8MM_VIDEO_PLL1_BYPASS], clks[IMX8MM_VIDEO_PLL1]); + clk_set_parent(clks[IMX8MM_DRAM_PLL_BYPASS], clks[IMX8MM_DRAM_PLL]); + clk_set_parent(clks[IMX8MM_GPU_PLL_BYPASS], clks[IMX8MM_GPU_PLL]); + clk_set_parent(clks[IMX8MM_VPU_PLL_BYPASS], clks[IMX8MM_VPU_PLL]); + clk_set_parent(clks[IMX8MM_ARM_PLL_BYPASS], clks[IMX8MM_ARM_PLL]); + clk_set_parent(clks[IMX8MM_SYS_PLL1_BYPASS], clks[IMX8MM_SYS_PLL1]); + clk_set_parent(clks[IMX8MM_SYS_PLL2_BYPASS], clks[IMX8MM_SYS_PLL2]); + clk_set_parent(clks[IMX8MM_SYS_PLL3_BYPASS], clks[IMX8MM_SYS_PLL3]); + + /* PLL out gate */ + clks[IMX8MM_AUDIO_PLL1_OUT] = imx_clk_gate("audio_pll1_out", "audio_pll1_bypass", base, 13); + clks[IMX8MM_AUDIO_PLL2_OUT] = imx_clk_gate("audio_pll2_out", "audio_pll2_bypass", base + 0x14, 13); + clks[IMX8MM_VIDEO_PLL1_OUT] = imx_clk_gate("video_pll1_out", "video_pll1_bypass", base + 0x28, 13); + clks[IMX8MM_DRAM_PLL_OUT] = imx_clk_gate("dram_pll_out", "dram_pll_bypass", base + 0x50, 13); + clks[IMX8MM_GPU_PLL_OUT] = imx_clk_gate("gpu_pll_out", "gpu_pll_bypass", base + 0x64, 13); + clks[IMX8MM_VPU_PLL_OUT] = imx_clk_gate("vpu_pll_out", "vpu_pll_bypass", base + 0x74, 13); + clks[IMX8MM_ARM_PLL_OUT] = imx_clk_gate("arm_pll_out", "arm_pll_bypass", base + 0x84, 13); + clks[IMX8MM_SYS_PLL1_OUT] = imx_clk_gate("sys_pll1_out", "sys_pll1_bypass", base + 0x94, 13); + clks[IMX8MM_SYS_PLL2_OUT] = imx_clk_gate("sys_pll2_out", "sys_pll2_bypass", base + 0x104, 13); + clks[IMX8MM_SYS_PLL3_OUT] = imx_clk_gate("sys_pll3_out", "sys_pll3_bypass", base + 0x114, 13); + + /* SYS PLL fixed output */ + clks[IMX8MM_SYS_PLL1_40M] = imx_clk_fixed_factor("sys_pll1_40m", "sys_pll1_out", 1, 20); + clks[IMX8MM_SYS_PLL1_80M] = imx_clk_fixed_factor("sys_pll1_80m", "sys_pll1_out", 1, 10); + clks[IMX8MM_SYS_PLL1_100M] = imx_clk_fixed_factor("sys_pll1_100m", "sys_pll1_out", 1, 8); + clks[IMX8MM_SYS_PLL1_133M] = imx_clk_fixed_factor("sys_pll1_133m", "sys_pll1_out", 1, 6); + clks[IMX8MM_SYS_PLL1_160M] = imx_clk_fixed_factor("sys_pll1_160m", "sys_pll1_out", 1, 5); + clks[IMX8MM_SYS_PLL1_200M] = imx_clk_fixed_factor("sys_pll1_200m", "sys_pll1_out", 1, 4); + clks[IMX8MM_SYS_PLL1_266M] = imx_clk_fixed_factor("sys_pll1_266m", "sys_pll1_out", 1, 3); + clks[IMX8MM_SYS_PLL1_400M] = imx_clk_fixed_factor("sys_pll1_400m", "sys_pll1_out", 1, 2); + clks[IMX8MM_SYS_PLL1_800M] = imx_clk_fixed_factor("sys_pll1_800m", "sys_pll1_out", 1, 1); + + clks[IMX8MM_SYS_PLL2_50M] = imx_clk_fixed_factor("sys_pll2_50m", "sys_pll2_out", 1, 20); + clks[IMX8MM_SYS_PLL2_100M] = imx_clk_fixed_factor("sys_pll2_100m", "sys_pll2_out", 1, 10); + clks[IMX8MM_SYS_PLL2_125M] = imx_clk_fixed_factor("sys_pll2_125m", "sys_pll2_out", 1, 8); + clks[IMX8MM_SYS_PLL2_166M] = imx_clk_fixed_factor("sys_pll2_166m", "sys_pll2_out", 1, 6); + clks[IMX8MM_SYS_PLL2_200M] = imx_clk_fixed_factor("sys_pll2_200m", "sys_pll2_out", 1, 5); + clks[IMX8MM_SYS_PLL2_250M] = imx_clk_fixed_factor("sys_pll2_250m", "sys_pll2_out", 1, 4); + clks[IMX8MM_SYS_PLL2_333M] = imx_clk_fixed_factor("sys_pll2_333m", "sys_pll2_out", 1, 3); + clks[IMX8MM_SYS_PLL2_500M] = imx_clk_fixed_factor("sys_pll2_500m", "sys_pll2_out", 1, 2); + clks[IMX8MM_SYS_PLL2_1000M] = imx_clk_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1); + + np = ccm_node; + base = of_iomap(np, 0); + if (WARN_ON(!base)) + return -ENOMEM; + + /* Core Slice */ + clks[IMX8MM_CLK_A53_SRC] = imx_clk_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mm_a53_sels, ARRAY_SIZE(imx8mm_a53_sels)); + clks[IMX8MM_CLK_M4_SRC] = imx_clk_mux2("arm_m4_src", base + 0x8080, 24, 3, imx8mm_m4_sels, ARRAY_SIZE(imx8mm_m4_sels)); + clks[IMX8MM_CLK_VPU_SRC] = imx_clk_mux2("vpu_src", base + 0x8100, 24, 3, imx8mm_vpu_sels, ARRAY_SIZE(imx8mm_vpu_sels)); + clks[IMX8MM_CLK_GPU3D_SRC] = imx_clk_mux2("gpu3d_src", base + 0x8180, 24, 3, imx8mm_gpu3d_sels, ARRAY_SIZE(imx8mm_gpu3d_sels)); + clks[IMX8MM_CLK_GPU2D_SRC] = imx_clk_mux2("gpu2d_src", base + 0x8200, 24, 3, imx8mm_gpu2d_sels, ARRAY_SIZE(imx8mm_gpu2d_sels)); + clks[IMX8MM_CLK_A53_CG] = imx_clk_gate3("arm_a53_cg", "arm_a53_src", base + 0x8000, 28); + clks[IMX8MM_CLK_M4_CG] = imx_clk_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28); + clks[IMX8MM_CLK_VPU_CG] = imx_clk_gate3("vpu_cg", "vpu_src", base + 0x8100, 28); + clks[IMX8MM_CLK_GPU3D_CG] = imx_clk_gate3("gpu3d_cg", "gpu3d_src", base + 0x8180, 28); + clks[IMX8MM_CLK_GPU2D_CG] = imx_clk_gate3("gpu2d_cg", "gpu2d_src", base + 0x8200, 28); + clks[IMX8MM_CLK_A53_DIV] = imx_clk_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3); + clks[IMX8MM_CLK_M4_DIV] = imx_clk_divider2("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3); + clks[IMX8MM_CLK_VPU_DIV] = imx_clk_divider2("vpu_div", "vpu_cg", base + 0x8100, 0, 3); + clks[IMX8MM_CLK_GPU3D_DIV] = imx_clk_divider2("gpu3d_div", "gpu3d_cg", base + 0x8180, 0, 3); + clks[IMX8MM_CLK_GPU2D_DIV] = imx_clk_divider2("gpu2d_div", "gpu2d_cg", base + 0x8200, 0, 3); + + /* BUS */ + clks[IMX8MM_CLK_MAIN_AXI] = imx8m_clk_composite_critical("main_axi", imx8mm_main_axi_sels, base + 0x8800); + clks[IMX8MM_CLK_ENET_AXI] = imx8m_clk_composite("enet_axi", imx8mm_enet_axi_sels, base + 0x8880); + clks[IMX8MM_CLK_NAND_USDHC_BUS] = imx8m_clk_composite_critical("nand_usdhc_bus", imx8mm_nand_usdhc_sels, base + 0x8900); + clks[IMX8MM_CLK_VPU_BUS] = imx8m_clk_composite("vpu_bus", imx8mm_vpu_bus_sels, base + 0x8980); + clks[IMX8MM_CLK_DISP_AXI] = imx8m_clk_composite("disp_axi", imx8mm_disp_axi_sels, base + 0x8a00); + clks[IMX8MM_CLK_DISP_APB] = imx8m_clk_composite("disp_apb", imx8mm_disp_apb_sels, base + 0x8a80); + clks[IMX8MM_CLK_DISP_RTRM] = imx8m_clk_composite("disp_rtrm", imx8mm_disp_rtrm_sels, base + 0x8b00); + clks[IMX8MM_CLK_USB_BUS] = imx8m_clk_composite("usb_bus", imx8mm_usb_bus_sels, base + 0x8b80); + clks[IMX8MM_CLK_GPU_AXI] = imx8m_clk_composite("gpu_axi", imx8mm_gpu_axi_sels, base + 0x8c00); + clks[IMX8MM_CLK_GPU_AHB] = imx8m_clk_composite("gpu_ahb", imx8mm_gpu_ahb_sels, base + 0x8c80); + clks[IMX8MM_CLK_NOC] = imx8m_clk_composite_critical("noc", imx8mm_noc_sels, base + 0x8d00); + clks[IMX8MM_CLK_NOC_APB] = imx8m_clk_composite_critical("noc_apb", imx8mm_noc_apb_sels, base + 0x8d80); + + /* AHB */ + clks[IMX8MM_CLK_AHB] = imx8m_clk_composite_critical("ahb", imx8mm_ahb_sels, base + 0x9000); + clks[IMX8MM_CLK_AUDIO_AHB] = imx8m_clk_composite("audio_ahb", imx8mm_audio_ahb_sels, base + 0x9100); + + /* IPG */ + clks[IMX8MM_CLK_IPG_ROOT] = imx_clk_divider2("ipg_root", "ahb", base + 0x9080, 0, 1); + clks[IMX8MM_CLK_IPG_AUDIO_ROOT] = imx_clk_divider2("ipg_audio_root", "audio_ahb", base + 0x9180, 0, 1); + + /* IP */ + clks[IMX8MM_CLK_DRAM_ALT] = imx8m_clk_composite("dram_alt", imx8mm_dram_alt_sels, base + 0xa000); + clks[IMX8MM_CLK_DRAM_APB] = imx8m_clk_composite("dram_apb", imx8mm_dram_apb_sels, base + 0xa080); + clks[IMX8MM_CLK_VPU_G1] = imx8m_clk_composite("vpu_g1", imx8mm_vpu_g1_sels, base + 0xa100); + clks[IMX8MM_CLK_VPU_G2] = imx8m_clk_composite("vpu_g2", imx8mm_vpu_g2_sels, base + 0xa180); + clks[IMX8MM_CLK_DISP_DTRC] = imx8m_clk_composite("disp_dtrc", imx8mm_disp_dtrc_sels, base + 0xa200); + clks[IMX8MM_CLK_DISP_DC8000] = imx8m_clk_composite("disp_dc8000", imx8mm_disp_dc8000_sels, base + 0xa280); + clks[IMX8MM_CLK_PCIE1_CTRL] = imx8m_clk_composite("pcie1_ctrl", imx8mm_pcie1_ctrl_sels, base + 0xa300); + clks[IMX8MM_CLK_PCIE1_PHY] = imx8m_clk_composite("pcie1_phy", imx8mm_pcie1_phy_sels, base + 0xa380); + clks[IMX8MM_CLK_PCIE1_AUX] = imx8m_clk_composite("pcie1_aux", imx8mm_pcie1_aux_sels, base + 0xa400); + clks[IMX8MM_CLK_DC_PIXEL] = imx8m_clk_composite("dc_pixel", imx8mm_dc_pixel_sels, base + 0xa480); + clks[IMX8MM_CLK_LCDIF_PIXEL] = imx8m_clk_composite("lcdif_pixel", imx8mm_lcdif_pixel_sels, base + 0xa500); + clks[IMX8MM_CLK_SAI1] = imx8m_clk_composite("sai1", imx8mm_sai1_sels, base + 0xa580); + clks[IMX8MM_CLK_SAI2] = imx8m_clk_composite("sai2", imx8mm_sai2_sels, base + 0xa600); + clks[IMX8MM_CLK_SAI3] = imx8m_clk_composite("sai3", imx8mm_sai3_sels, base + 0xa680); + clks[IMX8MM_CLK_SAI4] = imx8m_clk_composite("sai4", imx8mm_sai4_sels, base + 0xa700); + clks[IMX8MM_CLK_SAI5] = imx8m_clk_composite("sai5", imx8mm_sai5_sels, base + 0xa780); + clks[IMX8MM_CLK_SAI6] = imx8m_clk_composite("sai6", imx8mm_sai6_sels, base + 0xa800); + clks[IMX8MM_CLK_SPDIF1] = imx8m_clk_composite("spdif1", imx8mm_spdif1_sels, base + 0xa880); + clks[IMX8MM_CLK_SPDIF2] = imx8m_clk_composite("spdif2", imx8mm_spdif2_sels, base + 0xa900); + clks[IMX8MM_CLK_ENET_REF] = imx8m_clk_composite("enet_ref", imx8mm_enet_ref_sels, base + 0xa980); + clks[IMX8MM_CLK_ENET_TIMER] = imx8m_clk_composite("enet_timer", imx8mm_enet_timer_sels, base + 0xaa00); + clks[IMX8MM_CLK_ENET_PHY_REF] = imx8m_clk_composite("enet_phy", imx8mm_enet_phy_sels, base + 0xaa80); + clks[IMX8MM_CLK_NAND] = imx8m_clk_composite("nand", imx8mm_nand_sels, base + 0xab00); + clks[IMX8MM_CLK_QSPI] = imx8m_clk_composite("qspi", imx8mm_qspi_sels, base + 0xab80); + clks[IMX8MM_CLK_USDHC1] = imx8m_clk_composite("usdhc1", imx8mm_usdhc1_sels, base + 0xac00); + clks[IMX8MM_CLK_USDHC2] = imx8m_clk_composite("usdhc2", imx8mm_usdhc2_sels, base + 0xac80); + clks[IMX8MM_CLK_I2C1] = imx8m_clk_composite("i2c1", imx8mm_i2c1_sels, base + 0xad00); + clks[IMX8MM_CLK_I2C2] = imx8m_clk_composite("i2c2", imx8mm_i2c2_sels, base + 0xad80); + clks[IMX8MM_CLK_I2C3] = imx8m_clk_composite("i2c3", imx8mm_i2c3_sels, base + 0xae00); + clks[IMX8MM_CLK_I2C4] = imx8m_clk_composite("i2c4", imx8mm_i2c4_sels, base + 0xae80); + clks[IMX8MM_CLK_UART1] = imx8m_clk_composite("uart1", imx8mm_uart1_sels, base + 0xaf00); + clks[IMX8MM_CLK_UART2] = imx8m_clk_composite("uart2", imx8mm_uart2_sels, base + 0xaf80); + clks[IMX8MM_CLK_UART3] = imx8m_clk_composite("uart3", imx8mm_uart3_sels, base + 0xb000); + clks[IMX8MM_CLK_UART4] = imx8m_clk_composite("uart4", imx8mm_uart4_sels, base + 0xb080); + clks[IMX8MM_CLK_USB_CORE_REF] = imx8m_clk_composite("usb_core_ref", imx8mm_usb_core_sels, base + 0xb100); + clks[IMX8MM_CLK_USB_PHY_REF] = imx8m_clk_composite("usb_phy_ref", imx8mm_usb_phy_sels, base + 0xb180); + clks[IMX8MM_CLK_ECSPI1] = imx8m_clk_composite("ecspi1", imx8mm_ecspi1_sels, base + 0xb280); + clks[IMX8MM_CLK_ECSPI2] = imx8m_clk_composite("ecspi2", imx8mm_ecspi2_sels, base + 0xb300); + clks[IMX8MM_CLK_PWM1] = imx8m_clk_composite("pwm1", imx8mm_pwm1_sels, base + 0xb380); + clks[IMX8MM_CLK_PWM2] = imx8m_clk_composite("pwm2", imx8mm_pwm2_sels, base + 0xb400); + clks[IMX8MM_CLK_PWM3] = imx8m_clk_composite("pwm3", imx8mm_pwm3_sels, base + 0xb480); + clks[IMX8MM_CLK_PWM4] = imx8m_clk_composite("pwm4", imx8mm_pwm4_sels, base + 0xb500); + clks[IMX8MM_CLK_GPT1] = imx8m_clk_composite("gpt1", imx8mm_gpt1_sels, base + 0xb580); + clks[IMX8MM_CLK_WDOG] = imx8m_clk_composite("wdog", imx8mm_wdog_sels, base + 0xb900); + clks[IMX8MM_CLK_WRCLK] = imx8m_clk_composite("wrclk", imx8mm_wrclk_sels, base + 0xb980); + clks[IMX8MM_CLK_CLKO1] = imx8m_clk_composite("clko1", imx8mm_clko1_sels, base + 0xba00); + clks[IMX8MM_CLK_DSI_CORE] = imx8m_clk_composite("dsi_core", imx8mm_dsi_core_sels, base + 0xbb00); + clks[IMX8MM_CLK_DSI_PHY_REF] = imx8m_clk_composite("dsi_phy_ref", imx8mm_dsi_phy_sels, base + 0xbb80); + clks[IMX8MM_CLK_DSI_DBI] = imx8m_clk_composite("dsi_dbi", imx8mm_dsi_dbi_sels, base + 0xbc00); + clks[IMX8MM_CLK_USDHC3] = imx8m_clk_composite("usdhc3", imx8mm_usdhc3_sels, base + 0xbc80); + clks[IMX8MM_CLK_CSI1_CORE] = imx8m_clk_composite("csi1_core", imx8mm_csi1_core_sels, base + 0xbd00); + clks[IMX8MM_CLK_CSI1_PHY_REF] = imx8m_clk_composite("csi1_phy_ref", imx8mm_csi1_phy_sels, base + 0xbd80); + clks[IMX8MM_CLK_CSI1_ESC] = imx8m_clk_composite("csi1_esc", imx8mm_csi1_esc_sels, base + 0xbe00); + clks[IMX8MM_CLK_CSI2_CORE] = imx8m_clk_composite("csi2_core", imx8mm_csi2_core_sels, base + 0xbe80); + clks[IMX8MM_CLK_CSI2_PHY_REF] = imx8m_clk_composite("csi2_phy_ref", imx8mm_csi2_phy_sels, base + 0xbf00); + clks[IMX8MM_CLK_CSI2_ESC] = imx8m_clk_composite("csi2_esc", imx8mm_csi2_esc_sels, base + 0xbf80); + clks[IMX8MM_CLK_PCIE2_CTRL] = imx8m_clk_composite("pcie2_ctrl", imx8mm_pcie2_ctrl_sels, base + 0xc000); + clks[IMX8MM_CLK_PCIE2_PHY] = imx8m_clk_composite("pcie2_phy", imx8mm_pcie2_phy_sels, base + 0xc080); + clks[IMX8MM_CLK_PCIE2_AUX] = imx8m_clk_composite("pcie2_aux", imx8mm_pcie2_aux_sels, base + 0xc100); + clks[IMX8MM_CLK_ECSPI3] = imx8m_clk_composite("ecspi3", imx8mm_ecspi3_sels, base + 0xc180); + clks[IMX8MM_CLK_PDM] = imx8m_clk_composite("pdm", imx8mm_pdm_sels, base + 0xc200); + clks[IMX8MM_CLK_VPU_H1] = imx8m_clk_composite("vpu_h1", imx8mm_vpu_h1_sels, base + 0xc280); + + /* CCGR */ + clks[IMX8MM_CLK_ECSPI1_ROOT] = imx_clk_gate4("ecspi1_root_clk", "ecspi1", base + 0x4070, 0); + clks[IMX8MM_CLK_ECSPI2_ROOT] = imx_clk_gate4("ecspi2_root_clk", "ecspi2", base + 0x4080, 0); + clks[IMX8MM_CLK_ECSPI3_ROOT] = imx_clk_gate4("ecspi3_root_clk", "ecspi3", base + 0x4090, 0); + clks[IMX8MM_CLK_ENET1_ROOT] = imx_clk_gate4("enet1_root_clk", "enet_axi", base + 0x40a0, 0); + clks[IMX8MM_CLK_GPT1_ROOT] = imx_clk_gate4("gpt1_root_clk", "gpt1", base + 0x4100, 0); + clks[IMX8MM_CLK_I2C1_ROOT] = imx_clk_gate4("i2c1_root_clk", "i2c1", base + 0x4170, 0); + clks[IMX8MM_CLK_I2C2_ROOT] = imx_clk_gate4("i2c2_root_clk", "i2c2", base + 0x4180, 0); + clks[IMX8MM_CLK_I2C3_ROOT] = imx_clk_gate4("i2c3_root_clk", "i2c3", base + 0x4190, 0); + clks[IMX8MM_CLK_I2C4_ROOT] = imx_clk_gate4("i2c4_root_clk", "i2c4", base + 0x41a0, 0); + clks[IMX8MM_CLK_MU_ROOT] = imx_clk_gate4("mu_root_clk", "ipg_root", base + 0x4210, 0); + clks[IMX8MM_CLK_OCOTP_ROOT] = imx_clk_gate4("ocotp_root_clk", "ipg_root", base + 0x4220, 0); + clks[IMX8MM_CLK_PCIE1_ROOT] = imx_clk_gate4("pcie1_root_clk", "pcie1_ctrl", base + 0x4250, 0); + clks[IMX8MM_CLK_PWM1_ROOT] = imx_clk_gate4("pwm1_root_clk", "pwm1", base + 0x4280, 0); + clks[IMX8MM_CLK_PWM2_ROOT] = imx_clk_gate4("pwm2_root_clk", "pwm2", base + 0x4290, 0); + clks[IMX8MM_CLK_PWM3_ROOT] = imx_clk_gate4("pwm3_root_clk", "pwm3", base + 0x42a0, 0); + clks[IMX8MM_CLK_PWM4_ROOT] = imx_clk_gate4("pwm4_root_clk", "pwm4", base + 0x42b0, 0); + clks[IMX8MM_CLK_QSPI_ROOT] = imx_clk_gate4("qspi_root_clk", "qspi", base + 0x42f0, 0); + clks[IMX8MM_CLK_NAND_ROOT] = imx_clk_gate2_shared2("nand_root_clk", "nand", base + 0x4300, 0, &share_count_nand); + clks[IMX8MM_CLK_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_bus", base + 0x4300, 0, &share_count_nand); + clks[IMX8MM_CLK_SAI1_ROOT] = imx_clk_gate2_shared2("sai1_root_clk", "sai1", base + 0x4330, 0, &share_count_sai1); + clks[IMX8MM_CLK_SAI1_IPG] = imx_clk_gate2_shared2("sai1_ipg_clk", "ipg_audio_root", base + 0x4330, 0, &share_count_sai1); + clks[IMX8MM_CLK_SAI2_ROOT] = imx_clk_gate2_shared2("sai2_root_clk", "sai2", base + 0x4340, 0, &share_count_sai2); + clks[IMX8MM_CLK_SAI2_IPG] = imx_clk_gate2_shared2("sai2_ipg_clk", "ipg_audio_root", base + 0x4340, 0, &share_count_sai2); + clks[IMX8MM_CLK_SAI3_ROOT] = imx_clk_gate2_shared2("sai3_root_clk", "sai3", base + 0x4350, 0, &share_count_sai3); + clks[IMX8MM_CLK_SAI3_IPG] = imx_clk_gate2_shared2("sai3_ipg_clk", "ipg_audio_root", base + 0x4350, 0, &share_count_sai3); + clks[IMX8MM_CLK_SAI4_ROOT] = imx_clk_gate2_shared2("sai4_root_clk", "sai4", base + 0x4360, 0, &share_count_sai4); + clks[IMX8MM_CLK_SAI4_IPG] = imx_clk_gate2_shared2("sai4_ipg_clk", "ipg_audio_root", base + 0x4360, 0, &share_count_sai4); + clks[IMX8MM_CLK_SAI5_ROOT] = imx_clk_gate2_shared2("sai5_root_clk", "sai5", base + 0x4370, 0, &share_count_sai5); + clks[IMX8MM_CLK_SAI5_IPG] = imx_clk_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5); + clks[IMX8MM_CLK_SAI6_ROOT] = imx_clk_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6); + clks[IMX8MM_CLK_SAI6_IPG] = imx_clk_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6); + clks[IMX8MM_CLK_UART1_ROOT] = imx_clk_gate4("uart1_root_clk", "uart1", base + 0x4490, 0); + clks[IMX8MM_CLK_UART2_ROOT] = imx_clk_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0); + clks[IMX8MM_CLK_UART3_ROOT] = imx_clk_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0); + clks[IMX8MM_CLK_UART4_ROOT] = imx_clk_gate4("uart4_root_clk", "uart4", base + 0x44c0, 0); + clks[IMX8MM_CLK_USB1_CTRL_ROOT] = imx_clk_gate4("usb1_ctrl_root_clk", "usb_core_ref", base + 0x44d0, 0); + clks[IMX8MM_CLK_GPU3D_ROOT] = imx_clk_gate4("gpu3d_root_clk", "gpu3d_div", base + 0x44f0, 0); + clks[IMX8MM_CLK_USDHC1_ROOT] = imx_clk_gate4("usdhc1_root_clk", "usdhc1", base + 0x4510, 0); + clks[IMX8MM_CLK_USDHC2_ROOT] = imx_clk_gate4("usdhc2_root_clk", "usdhc2", base + 0x4520, 0); + clks[IMX8MM_CLK_WDOG1_ROOT] = imx_clk_gate4("wdog1_root_clk", "wdog", base + 0x4530, 0); + clks[IMX8MM_CLK_WDOG2_ROOT] = imx_clk_gate4("wdog2_root_clk", "wdog", base + 0x4540, 0); + clks[IMX8MM_CLK_WDOG3_ROOT] = imx_clk_gate4("wdog3_root_clk", "wdog", base + 0x4550, 0); + clks[IMX8MM_CLK_VPU_G1_ROOT] = imx_clk_gate4("vpu_g1_root_clk", "vpu_g1", base + 0x4560, 0); + clks[IMX8MM_CLK_GPU_BUS_ROOT] = imx_clk_gate4("gpu_root_clk", "gpu_axi", base + 0x4570, 0); + clks[IMX8MM_CLK_VPU_H1_ROOT] = imx_clk_gate4("vpu_h1_root_clk", "vpu_h1", base + 0x4590, 0); + clks[IMX8MM_CLK_VPU_G2_ROOT] = imx_clk_gate4("vpu_g2_root_clk", "vpu_g2", base + 0x45a0, 0); + clks[IMX8MM_CLK_PDM_ROOT] = imx_clk_gate2_shared2("pdm_root_clk", "pdm", base + 0x45b0, 0, &share_count_pdm); + clks[IMX8MM_CLK_PDM_IPG] = imx_clk_gate2_shared2("pdm_ipg_clk", "ipg_audio_root", base + 0x45b0, 0, &share_count_pdm); + clks[IMX8MM_CLK_DISP_ROOT] = imx_clk_gate2_shared2("disp_root_clk", "disp_dc8000", base + 0x45d0, 0, &share_count_dcss); + clks[IMX8MM_CLK_DISP_AXI_ROOT] = imx_clk_gate2_shared2("disp_axi_root_clk", "disp_axi", base + 0x45d0, 0, &share_count_dcss); + clks[IMX8MM_CLK_DISP_APB_ROOT] = imx_clk_gate2_shared2("disp_apb_root_clk", "disp_apb", base + 0x45d0, 0, &share_count_dcss); + clks[IMX8MM_CLK_DISP_RTRM_ROOT] = imx_clk_gate2_shared2("disp_rtrm_root_clk", "disp_rtrm", base + 0x45d0, 0, &share_count_dcss); + clks[IMX8MM_CLK_USDHC3_ROOT] = imx_clk_gate4("usdhc3_root_clk", "usdhc3", base + 0x45e0, 0); + clks[IMX8MM_CLK_TMU_ROOT] = imx_clk_gate4("tmu_root_clk", "ipg_root", base + 0x4620, 0); + clks[IMX8MM_CLK_VPU_DEC_ROOT] = imx_clk_gate4("vpu_dec_root_clk", "vpu_bus", base + 0x4630, 0); + clks[IMX8MM_CLK_SDMA1_ROOT] = imx_clk_gate4("sdma1_clk", "ipg_root", base + 0x43a0, 0); + clks[IMX8MM_CLK_SDMA2_ROOT] = imx_clk_gate4("sdma2_clk", "ipg_audio_root", base + 0x43b0, 0); + clks[IMX8MM_CLK_SDMA3_ROOT] = imx_clk_gate4("sdma3_clk", "ipg_audio_root", base + 0x45f0, 0); + clks[IMX8MM_CLK_GPU2D_ROOT] = imx_clk_gate4("gpu2d_root_clk", "gpu2d_div", base + 0x4660, 0); + clks[IMX8MM_CLK_CSI1_ROOT] = imx_clk_gate4("csi1_root_clk", "csi1_core", base + 0x4650, 0); + + clks[IMX8MM_CLK_GPT_3M] = imx_clk_fixed_factor("gpt_3m", "osc_24m", 1, 8); + + clks[IMX8MM_CLK_DRAM_ALT_ROOT] = imx_clk_fixed_factor("dram_alt_root", "dram_alt", 1, 4); + clks[IMX8MM_CLK_DRAM_CORE] = imx_clk_mux2_flags("dram_core_clk", base + 0x9800, 24, 1, imx8mm_dram_core_sels, ARRAY_SIZE(imx8mm_dram_core_sels), CLK_IS_CRITICAL); + + clks[IMX8MM_CLK_ARM] = imx_clk_cpu("arm", "arm_a53_div", + clks[IMX8MM_CLK_A53_DIV], + clks[IMX8MM_CLK_A53_SRC], + clks[IMX8MM_ARM_PLL_OUT], + clks[IMX8MM_CLK_24M]); + + imx_check_clocks(clks, ARRAY_SIZE(clks)); + + clk_data.clks = clks; + clk_data.clk_num = ARRAY_SIZE(clks); + ret = of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); + if (ret < 0) { + pr_err("failed to register clks for i.MX8MM\n"); + return -EINVAL; + } + + imx_register_uart_clocks(uart_clks); + + return 0; +} +CLK_OF_DECLARE_DRIVER(imx8mm, "fsl,imx8mm-ccm", imx8mm_clocks_init); diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c index 26b57f43ccc3b..a9b3888aef0c2 100644 --- a/drivers/clk/imx/clk-imx8mq.c +++ b/drivers/clk/imx/clk-imx8mq.c @@ -26,246 +26,246 @@ static u32 share_count_nand; static struct clk *clks[IMX8MQ_CLK_END]; -static const char *pll_ref_sels[] = { "osc_25m", "osc_27m", "dummy", "dummy", }; -static const char *arm_pll_bypass_sels[] = {"arm_pll", "arm_pll_ref_sel", }; -static const char *gpu_pll_bypass_sels[] = {"gpu_pll", "gpu_pll_ref_sel", }; -static const char *vpu_pll_bypass_sels[] = {"vpu_pll", "vpu_pll_ref_sel", }; -static const char *audio_pll1_bypass_sels[] = {"audio_pll1", "audio_pll1_ref_sel", }; -static const char *audio_pll2_bypass_sels[] = {"audio_pll2", "audio_pll2_ref_sel", }; -static const char *video_pll1_bypass_sels[] = {"video_pll1", "video_pll1_ref_sel", }; - -static const char *sys1_pll1_out_sels[] = {"sys1_pll1", "sys1_pll1_ref_sel", }; -static const char *sys2_pll1_out_sels[] = {"sys2_pll1", "sys1_pll1_ref_sel", }; -static const char *sys3_pll1_out_sels[] = {"sys3_pll1", "sys3_pll1_ref_sel", }; -static const char *dram_pll1_out_sels[] = {"dram_pll1", "dram_pll1_ref_sel", }; - -static const char *sys1_pll2_out_sels[] = {"sys1_pll2_div", "sys1_pll1_ref_sel", }; -static const char *sys2_pll2_out_sels[] = {"sys2_pll2_div", "sys2_pll1_ref_sel", }; -static const char *sys3_pll2_out_sels[] = {"sys3_pll2_div", "sys2_pll1_ref_sel", }; -static const char *dram_pll2_out_sels[] = {"dram_pll2_div", "dram_pll1_ref_sel", }; +static const char * const pll_ref_sels[] = { "osc_25m", "osc_27m", "dummy", "dummy", }; +static const char * const arm_pll_bypass_sels[] = {"arm_pll", "arm_pll_ref_sel", }; +static const char * const gpu_pll_bypass_sels[] = {"gpu_pll", "gpu_pll_ref_sel", }; +static const char * const vpu_pll_bypass_sels[] = {"vpu_pll", "vpu_pll_ref_sel", }; +static const char * const audio_pll1_bypass_sels[] = {"audio_pll1", "audio_pll1_ref_sel", }; +static const char * const audio_pll2_bypass_sels[] = {"audio_pll2", "audio_pll2_ref_sel", }; +static const char * const video_pll1_bypass_sels[] = {"video_pll1", "video_pll1_ref_sel", }; + +static const char * const sys1_pll_out_sels[] = {"sys1_pll1_ref_sel", }; +static const char * const sys2_pll_out_sels[] = {"sys1_pll1_ref_sel", "sys2_pll1_ref_sel", }; +static const char * const sys3_pll_out_sels[] = {"sys3_pll1_ref_sel", "sys2_pll1_ref_sel", }; +static const char * const dram_pll_out_sels[] = {"dram_pll1_ref_sel", }; /* CCM ROOT */ -static const char *imx8mq_a53_sels[] = {"osc_25m", "arm_pll_out", "sys2_pll_500m", "sys2_pll_1000m", +static const char * const imx8mq_a53_sels[] = {"osc_25m", "arm_pll_out", "sys2_pll_500m", "sys2_pll_1000m", "sys1_pll_800m", "sys1_pll_400m", "audio_pll1_out", "sys3_pll2_out", }; -static const char *imx8mq_vpu_sels[] = {"osc_25m", "arm_pll_out", "sys2_pll_500m", "sys2_pll_1000m", +static const char * const imx8mq_arm_m4_sels[] = {"osc_25m", "sys2_pll_200m", "sys2_pll_250m", "sys1_pll_266m", + "sys1_pll_800m", "audio_pll1_out", "video_pll1_out", "sys3_pll2_out", }; + +static const char * const imx8mq_vpu_sels[] = {"osc_25m", "arm_pll_out", "sys2_pll_500m", "sys2_pll_1000m", "sys1_pll_800m", "sys1_pll_400m", "audio_pll1_out", "vpu_pll_out", }; -static const char *imx8mq_gpu_core_sels[] = {"osc_25m", "gpu_pll_out", "sys1_pll_800m", "sys3_pll2_out", +static const char * const imx8mq_gpu_core_sels[] = {"osc_25m", "gpu_pll_out", "sys1_pll_800m", "sys3_pll2_out", "sys2_pll_1000m", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", }; -static const char *imx8mq_gpu_shader_sels[] = {"osc_25m", "gpu_pll_out", "sys1_pll_800m", "sys3_pll2_out", +static const char * const imx8mq_gpu_shader_sels[] = {"osc_25m", "gpu_pll_out", "sys1_pll_800m", "sys3_pll2_out", "sys2_pll_1000m", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", }; -static const char *imx8mq_main_axi_sels[] = {"osc_25m", "sys2_pll_333m", "sys1_pll_800m", "sys2_pll_250m", +static const char * const imx8mq_main_axi_sels[] = {"osc_25m", "sys2_pll_333m", "sys1_pll_800m", "sys2_pll_250m", "sys2_pll_1000m", "audio_pll1_out", "video_pll1_out", "sys1_pll_100m",}; -static const char *imx8mq_enet_axi_sels[] = {"osc_25m", "sys1_pll_266m", "sys1_pll_800m", "sys2_pll_250m", +static const char * const imx8mq_enet_axi_sels[] = {"osc_25m", "sys1_pll_266m", "sys1_pll_800m", "sys2_pll_250m", "sys2_pll_200m", "audio_pll1_out", "video_pll1_out", "sys3_pll2_out", }; -static const char *imx8mq_nand_usdhc_sels[] = {"osc_25m", "sys1_pll_266m", "sys1_pll_800m", "sys2_pll_200m", +static const char * const imx8mq_nand_usdhc_sels[] = {"osc_25m", "sys1_pll_266m", "sys1_pll_800m", "sys2_pll_200m", "sys1_pll_133m", "sys3_pll2_out", "sys2_pll_250m", "audio_pll1_out", }; -static const char *imx8mq_vpu_bus_sels[] = {"osc_25m", "sys1_pll_800m", "vpu_pll_out", "audio_pll2_out", "sys3_pll2_out", "sys2_pll_1000m", "sys2_pll_200m", "sys1_pll_100m", }; +static const char * const imx8mq_vpu_bus_sels[] = {"osc_25m", "sys1_pll_800m", "vpu_pll_out", "audio_pll2_out", "sys3_pll2_out", "sys2_pll_1000m", "sys2_pll_200m", "sys1_pll_100m", }; -static const char *imx8mq_disp_axi_sels[] = {"osc_25m", "sys2_pll_125m", "sys1_pll_800m", "sys3_pll2_out", "sys1_pll_400m", "audio_pll2_out", "clk_ext1", "clk_ext4", }; +static const char * const imx8mq_disp_axi_sels[] = {"osc_25m", "sys2_pll_125m", "sys1_pll_800m", "sys3_pll2_out", "sys1_pll_400m", "audio_pll2_out", "clk_ext1", "clk_ext4", }; -static const char *imx8mq_disp_apb_sels[] = {"osc_25m", "sys2_pll_125m", "sys1_pll_800m", "sys3_pll2_out", +static const char * const imx8mq_disp_apb_sels[] = {"osc_25m", "sys2_pll_125m", "sys1_pll_800m", "sys3_pll2_out", "sys1_pll_40m", "audio_pll2_out", "clk_ext1", "clk_ext3", }; -static const char *imx8mq_disp_rtrm_sels[] = {"osc_25m", "sys1_pll_800m", "sys2_pll_200m", "sys1_pll_400m", +static const char * const imx8mq_disp_rtrm_sels[] = {"osc_25m", "sys1_pll_800m", "sys2_pll_200m", "sys1_pll_400m", "audio_pll1_out", "video_pll1_out", "clk_ext2", "clk_ext3", }; -static const char *imx8mq_usb_bus_sels[] = {"osc_25m", "sys2_pll_500m", "sys1_pll_800m", "sys2_pll_100m", +static const char * const imx8mq_usb_bus_sels[] = {"osc_25m", "sys2_pll_500m", "sys1_pll_800m", "sys2_pll_100m", "sys2_pll_200m", "clk_ext2", "clk_ext4", "audio_pll2_out", }; -static const char *imx8mq_gpu_axi_sels[] = {"osc_25m", "sys1_pll_800m", "gpu_pll_out", "sys3_pll2_out", "sys2_pll_1000m", +static const char * const imx8mq_gpu_axi_sels[] = {"osc_25m", "sys1_pll_800m", "gpu_pll_out", "sys3_pll2_out", "sys2_pll_1000m", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", }; -static const char *imx8mq_gpu_ahb_sels[] = {"osc_25m", "sys1_pll_800m", "gpu_pll_out", "sys3_pll2_out", "sys2_pll_1000m", +static const char * const imx8mq_gpu_ahb_sels[] = {"osc_25m", "sys1_pll_800m", "gpu_pll_out", "sys3_pll2_out", "sys2_pll_1000m", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", }; -static const char *imx8mq_noc_sels[] = {"osc_25m", "sys1_pll_800m", "sys3_pll2_out", "sys2_pll_1000m", "sys2_pll_500m", +static const char * const imx8mq_noc_sels[] = {"osc_25m", "sys1_pll_800m", "sys3_pll2_out", "sys2_pll_1000m", "sys2_pll_500m", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", }; -static const char *imx8mq_noc_apb_sels[] = {"osc_25m", "sys1_pll_400m", "sys3_pll2_out", "sys2_pll_333m", "sys2_pll_200m", +static const char * const imx8mq_noc_apb_sels[] = {"osc_25m", "sys1_pll_400m", "sys3_pll2_out", "sys2_pll_333m", "sys2_pll_200m", "sys1_pll_800m", "audio_pll1_out", "video_pll1_out", }; -static const char *imx8mq_ahb_sels[] = {"osc_25m", "sys1_pll_133m", "sys1_pll_800m", "sys1_pll_400m", +static const char * const imx8mq_ahb_sels[] = {"osc_25m", "sys1_pll_133m", "sys1_pll_800m", "sys1_pll_400m", "sys2_pll_125m", "sys3_pll2_out", "audio_pll1_out", "video_pll1_out", }; -static const char *imx8mq_audio_ahb_sels[] = {"osc_25m", "sys2_pll_500m", "sys1_pll_800m", "sys2_pll_1000m", +static const char * const imx8mq_audio_ahb_sels[] = {"osc_25m", "sys2_pll_500m", "sys1_pll_800m", "sys2_pll_1000m", "sys2_pll_166m", "sys3_pll2_out", "audio_pll1_out", "video_pll1_out", }; -static const char *imx8mq_dsi_ahb_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m", +static const char * const imx8mq_dsi_ahb_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "clk_ext3", "audio_pll2_out"}; -static const char *imx8mq_dram_alt_sels[] = {"osc_25m", "sys1_pll_800m", "sys1_pll_100m", "sys2_pll_500m", +static const char * const imx8mq_dram_alt_sels[] = {"osc_25m", "sys1_pll_800m", "sys1_pll_100m", "sys2_pll_500m", "sys2_pll_250m", "sys1_pll_400m", "audio_pll1_out", "sys1_pll_266m", }; -static const char *imx8mq_dram_apb_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys1_pll_160m", +static const char * const imx8mq_dram_apb_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys1_pll_160m", "sys1_pll_800m", "sys3_pll2_out", "sys2_pll_250m", "audio_pll2_out", }; -static const char *imx8mq_vpu_g1_sels[] = {"osc_25m", "vpu_pll_out", "sys1_pll_800m", "sys2_pll_1000m", "sys1_pll_100m", "sys2_pll_125m", "sys3_pll2_out", "audio_pll1_out", }; +static const char * const imx8mq_vpu_g1_sels[] = {"osc_25m", "vpu_pll_out", "sys1_pll_800m", "sys2_pll_1000m", "sys1_pll_100m", "sys2_pll_125m", "sys3_pll2_out", "audio_pll1_out", }; -static const char *imx8mq_vpu_g2_sels[] = {"osc_25m", "vpu_pll_out", "sys1_pll_800m", "sys2_pll_1000m", "sys1_pll_100m", "sys2_pll_125m", "sys3_pll2_out", "audio_pll1_out", }; +static const char * const imx8mq_vpu_g2_sels[] = {"osc_25m", "vpu_pll_out", "sys1_pll_800m", "sys2_pll_1000m", "sys1_pll_100m", "sys2_pll_125m", "sys3_pll2_out", "audio_pll1_out", }; -static const char *imx8mq_disp_dtrc_sels[] = {"osc_25m", "vpu_pll_out", "sys1_pll_800m", "sys2_pll_1000m", "sys1_pll_160m", "sys2_pll_100m", "sys3_pll2_out", "audio_pll2_out", }; +static const char * const imx8mq_disp_dtrc_sels[] = {"osc_25m", "vpu_pll_out", "sys1_pll_800m", "sys2_pll_1000m", "sys1_pll_160m", "sys2_pll_100m", "sys3_pll2_out", "audio_pll2_out", }; -static const char *imx8mq_disp_dc8000_sels[] = {"osc_25m", "vpu_pll_out", "sys1_pll_800m", "sys2_pll_1000m", "sys1_pll_160m", "sys2_pll_100m", "sys3_pll2_out", "audio_pll2_out", }; +static const char * const imx8mq_disp_dc8000_sels[] = {"osc_25m", "vpu_pll_out", "sys1_pll_800m", "sys2_pll_1000m", "sys1_pll_160m", "sys2_pll_100m", "sys3_pll2_out", "audio_pll2_out", }; -static const char *imx8mq_pcie1_ctrl_sels[] = {"osc_25m", "sys2_pll_250m", "sys2_pll_200m", "sys1_pll_266m", +static const char * const imx8mq_pcie1_ctrl_sels[] = {"osc_25m", "sys2_pll_250m", "sys2_pll_200m", "sys1_pll_266m", "sys1_pll_800m", "sys2_pll_500m", "sys2_pll_250m", "sys3_pll2_out", }; -static const char *imx8mq_pcie1_phy_sels[] = {"osc_25m", "sys2_pll_100m", "sys2_pll_500m", "clk_ext1", "clk_ext2", +static const char * const imx8mq_pcie1_phy_sels[] = {"osc_25m", "sys2_pll_100m", "sys2_pll_500m", "clk_ext1", "clk_ext2", "clk_ext3", "clk_ext4", }; -static const char *imx8mq_pcie1_aux_sels[] = {"osc_25m", "sys2_pll_200m", "sys2_pll_500m", "sys3_pll2_out", +static const char * const imx8mq_pcie1_aux_sels[] = {"osc_25m", "sys2_pll_200m", "sys2_pll_500m", "sys3_pll2_out", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_160m", "sys1_pll_200m", }; -static const char *imx8mq_dc_pixel_sels[] = {"osc_25m", "video_pll1_out", "audio_pll2_out", "audio_pll1_out", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "clk_ext4", }; +static const char * const imx8mq_dc_pixel_sels[] = {"osc_25m", "video_pll1_out", "audio_pll2_out", "audio_pll1_out", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "clk_ext4", }; -static const char *imx8mq_lcdif_pixel_sels[] = {"osc_25m", "video_pll1_out", "audio_pll2_out", "audio_pll1_out", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "clk_ext4", }; +static const char * const imx8mq_lcdif_pixel_sels[] = {"osc_25m", "video_pll1_out", "audio_pll2_out", "audio_pll1_out", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "clk_ext4", }; -static const char *imx8mq_sai1_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext1", "clk_ext2", }; +static const char * const imx8mq_sai1_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext1", "clk_ext2", }; -static const char *imx8mq_sai2_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext2", "clk_ext3", }; +static const char * const imx8mq_sai2_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext2", "clk_ext3", }; -static const char *imx8mq_sai3_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext3", "clk_ext4", }; +static const char * const imx8mq_sai3_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext3", "clk_ext4", }; -static const char *imx8mq_sai4_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext1", "clk_ext2", }; +static const char * const imx8mq_sai4_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext1", "clk_ext2", }; -static const char *imx8mq_sai5_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext2", "clk_ext3", }; +static const char * const imx8mq_sai5_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext2", "clk_ext3", }; -static const char *imx8mq_sai6_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext3", "clk_ext4", }; +static const char * const imx8mq_sai6_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext3", "clk_ext4", }; -static const char *imx8mq_spdif1_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext2", "clk_ext3", }; +static const char * const imx8mq_spdif1_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext2", "clk_ext3", }; -static const char *imx8mq_spdif2_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext3", "clk_ext4", }; +static const char * const imx8mq_spdif2_sels[] = {"osc_25m", "audio_pll1_out", "audio_pll2_out", "video_pll1_out", "sys1_pll_133m", "osc_27m", "clk_ext3", "clk_ext4", }; -static const char *imx8mq_enet_ref_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_500m", "sys2_pll_100m", +static const char * const imx8mq_enet_ref_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_500m", "sys2_pll_100m", "sys1_pll_160m", "audio_pll1_out", "video_pll1_out", "clk_ext4", }; -static const char *imx8mq_enet_timer_sels[] = {"osc_25m", "sys2_pll_100m", "audio_pll1_out", "clk_ext1", "clk_ext2", +static const char * const imx8mq_enet_timer_sels[] = {"osc_25m", "sys2_pll_100m", "audio_pll1_out", "clk_ext1", "clk_ext2", "clk_ext3", "clk_ext4", "video_pll1_out", }; -static const char *imx8mq_enet_phy_sels[] = {"osc_25m", "sys2_pll_50m", "sys2_pll_125m", "sys2_pll_500m", +static const char * const imx8mq_enet_phy_sels[] = {"osc_25m", "sys2_pll_50m", "sys2_pll_125m", "sys2_pll_500m", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", }; -static const char *imx8mq_nand_sels[] = {"osc_25m", "sys2_pll_500m", "audio_pll1_out", "sys1_pll_400m", +static const char * const imx8mq_nand_sels[] = {"osc_25m", "sys2_pll_500m", "audio_pll1_out", "sys1_pll_400m", "audio_pll2_out", "sys3_pll2_out", "sys2_pll_250m", "video_pll1_out", }; -static const char *imx8mq_qspi_sels[] = {"osc_25m", "sys1_pll_400m", "sys1_pll_800m", "sys2_pll_500m", +static const char * const imx8mq_qspi_sels[] = {"osc_25m", "sys1_pll_400m", "sys1_pll_800m", "sys2_pll_500m", "audio_pll2_out", "sys1_pll_266m", "sys3_pll2_out", "sys1_pll_100m", }; -static const char *imx8mq_usdhc1_sels[] = {"osc_25m", "sys1_pll_400m", "sys1_pll_800m", "sys2_pll_500m", +static const char * const imx8mq_usdhc1_sels[] = {"osc_25m", "sys1_pll_400m", "sys1_pll_800m", "sys2_pll_500m", "audio_pll2_out", "sys1_pll_266m", "sys3_pll2_out", "sys1_pll_100m", }; -static const char *imx8mq_usdhc2_sels[] = {"osc_25m", "sys1_pll_400m", "sys1_pll_800m", "sys2_pll_500m", +static const char * const imx8mq_usdhc2_sels[] = {"osc_25m", "sys1_pll_400m", "sys1_pll_800m", "sys2_pll_500m", "audio_pll2_out", "sys1_pll_266m", "sys3_pll2_out", "sys1_pll_100m", }; -static const char *imx8mq_i2c1_sels[] = {"osc_25m", "sys1_pll_160m", "sys2_pll_50m", "sys3_pll2_out", "audio_pll1_out", +static const char * const imx8mq_i2c1_sels[] = {"osc_25m", "sys1_pll_160m", "sys2_pll_50m", "sys3_pll2_out", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", "sys1_pll_133m", }; -static const char *imx8mq_i2c2_sels[] = {"osc_25m", "sys1_pll_160m", "sys2_pll_50m", "sys3_pll2_out", "audio_pll1_out", +static const char * const imx8mq_i2c2_sels[] = {"osc_25m", "sys1_pll_160m", "sys2_pll_50m", "sys3_pll2_out", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", "sys1_pll_133m", }; -static const char *imx8mq_i2c3_sels[] = {"osc_25m", "sys1_pll_160m", "sys2_pll_50m", "sys3_pll2_out", "audio_pll1_out", +static const char * const imx8mq_i2c3_sels[] = {"osc_25m", "sys1_pll_160m", "sys2_pll_50m", "sys3_pll2_out", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", "sys1_pll_133m", }; -static const char *imx8mq_i2c4_sels[] = {"osc_25m", "sys1_pll_160m", "sys2_pll_50m", "sys3_pll2_out", "audio_pll1_out", +static const char * const imx8mq_i2c4_sels[] = {"osc_25m", "sys1_pll_160m", "sys2_pll_50m", "sys3_pll2_out", "audio_pll1_out", "video_pll1_out", "audio_pll2_out", "sys1_pll_133m", }; -static const char *imx8mq_uart1_sels[] = {"osc_25m", "sys1_pll_80m", "sys2_pll_200m", "sys2_pll_100m", +static const char * const imx8mq_uart1_sels[] = {"osc_25m", "sys1_pll_80m", "sys2_pll_200m", "sys2_pll_100m", "sys3_pll2_out", "clk_ext2", "clk_ext4", "audio_pll2_out", }; -static const char *imx8mq_uart2_sels[] = {"osc_25m", "sys1_pll_80m", "sys2_pll_200m", "sys2_pll_100m", +static const char * const imx8mq_uart2_sels[] = {"osc_25m", "sys1_pll_80m", "sys2_pll_200m", "sys2_pll_100m", "sys3_pll2_out", "clk_ext2", "clk_ext3", "audio_pll2_out", }; -static const char *imx8mq_uart3_sels[] = {"osc_25m", "sys1_pll_80m", "sys2_pll_200m", "sys2_pll_100m", +static const char * const imx8mq_uart3_sels[] = {"osc_25m", "sys1_pll_80m", "sys2_pll_200m", "sys2_pll_100m", "sys3_pll2_out", "clk_ext2", "clk_ext4", "audio_pll2_out", }; -static const char *imx8mq_uart4_sels[] = {"osc_25m", "sys1_pll_80m", "sys2_pll_200m", "sys2_pll_100m", +static const char * const imx8mq_uart4_sels[] = {"osc_25m", "sys1_pll_80m", "sys2_pll_200m", "sys2_pll_100m", "sys3_pll2_out", "clk_ext2", "clk_ext3", "audio_pll2_out", }; -static const char *imx8mq_usb_core_sels[] = {"osc_25m", "sys1_pll_100m", "sys1_pll_40m", "sys2_pll_100m", +static const char * const imx8mq_usb_core_sels[] = {"osc_25m", "sys1_pll_100m", "sys1_pll_40m", "sys2_pll_100m", "sys2_pll_200m", "clk_ext2", "clk_ext3", "audio_pll2_out", }; -static const char *imx8mq_usb_phy_sels[] = {"osc_25m", "sys1_pll_100m", "sys1_pll_40m", "sys2_pll_100m", +static const char * const imx8mq_usb_phy_sels[] = {"osc_25m", "sys1_pll_100m", "sys1_pll_40m", "sys2_pll_100m", "sys2_pll_200m", "clk_ext2", "clk_ext3", "audio_pll2_out", }; -static const char *imx8mq_ecspi1_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys1_pll_160m", +static const char * const imx8mq_ecspi1_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys1_pll_160m", "sys1_pll_800m", "sys3_pll2_out", "sys2_pll_250m", "audio_pll2_out", }; -static const char *imx8mq_ecspi2_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys1_pll_160m", +static const char * const imx8mq_ecspi2_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys1_pll_160m", "sys1_pll_800m", "sys3_pll2_out", "sys2_pll_250m", "audio_pll2_out", }; -static const char *imx8mq_pwm1_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_160m", "sys1_pll_40m", +static const char * const imx8mq_pwm1_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_160m", "sys1_pll_40m", "sys3_pll2_out", "clk_ext1", "sys1_pll_80m", "video_pll1_out", }; -static const char *imx8mq_pwm2_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_160m", "sys1_pll_40m", +static const char * const imx8mq_pwm2_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_160m", "sys1_pll_40m", "sys3_pll2_out", "clk_ext1", "sys1_pll_80m", "video_pll1_out", }; -static const char *imx8mq_pwm3_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_160m", "sys1_pll_40m", +static const char * const imx8mq_pwm3_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_160m", "sys1_pll_40m", "sys3_pll2_out", "clk_ext2", "sys1_pll_80m", "video_pll1_out", }; -static const char *imx8mq_pwm4_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_160m", "sys1_pll_40m", +static const char * const imx8mq_pwm4_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_160m", "sys1_pll_40m", "sys3_pll2_out", "clk_ext2", "sys1_pll_80m", "video_pll1_out", }; -static const char *imx8mq_gpt1_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_400m", "sys1_pll_40m", +static const char * const imx8mq_gpt1_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_400m", "sys1_pll_40m", "sys1_pll_80m", "audio_pll1_out", "clk_ext1", }; -static const char *imx8mq_wdog_sels[] = {"osc_25m", "sys1_pll_133m", "sys1_pll_160m", "vpu_pll_out", +static const char * const imx8mq_wdog_sels[] = {"osc_25m", "sys1_pll_133m", "sys1_pll_160m", "vpu_pll_out", "sys2_pll_125m", "sys3_pll2_out", "sys1_pll_80m", "sys2_pll_166m", }; -static const char *imx8mq_wrclk_sels[] = {"osc_25m", "sys1_pll_40m", "vpu_pll_out", "sys3_pll2_out", "sys2_pll_200m", +static const char * const imx8mq_wrclk_sels[] = {"osc_25m", "sys1_pll_40m", "vpu_pll_out", "sys3_pll2_out", "sys2_pll_200m", "sys1_pll_266m", "sys2_pll_500m", "sys1_pll_100m", }; -static const char *imx8mq_dsi_core_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_250m", "sys1_pll_800m", +static const char * const imx8mq_dsi_core_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_250m", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "audio_pll2_out", "video_pll1_out", }; -static const char *imx8mq_dsi_phy_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_100m", "sys1_pll_800m", +static const char * const imx8mq_dsi_phy_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_100m", "sys1_pll_800m", "sys2_pll_1000m", "clk_ext2", "audio_pll2_out", "video_pll1_out", }; -static const char *imx8mq_dsi_dbi_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_100m", "sys1_pll_800m", +static const char * const imx8mq_dsi_dbi_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_100m", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "audio_pll2_out", "video_pll1_out", }; -static const char *imx8mq_dsi_esc_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m", +static const char * const imx8mq_dsi_esc_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "clk_ext3", "audio_pll2_out", }; -static const char *imx8mq_csi1_core_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_250m", "sys1_pll_800m", +static const char * const imx8mq_csi1_core_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_250m", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "audio_pll2_out", "video_pll1_out", }; -static const char *imx8mq_csi1_phy_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_100m", "sys1_pll_800m", +static const char * const imx8mq_csi1_phy_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_100m", "sys1_pll_800m", "sys2_pll_1000m", "clk_ext2", "audio_pll2_out", "video_pll1_out", }; -static const char *imx8mq_csi1_esc_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m", +static const char * const imx8mq_csi1_esc_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "clk_ext3", "audio_pll2_out", }; -static const char *imx8mq_csi2_core_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_250m", "sys1_pll_800m", +static const char * const imx8mq_csi2_core_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_250m", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "audio_pll2_out", "video_pll1_out", }; -static const char *imx8mq_csi2_phy_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_100m", "sys1_pll_800m", +static const char * const imx8mq_csi2_phy_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_100m", "sys1_pll_800m", "sys2_pll_1000m", "clk_ext2", "audio_pll2_out", "video_pll1_out", }; -static const char *imx8mq_csi2_esc_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m", +static const char * const imx8mq_csi2_esc_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll2_out", "clk_ext3", "audio_pll2_out", }; -static const char *imx8mq_pcie2_ctrl_sels[] = {"osc_25m", "sys2_pll_250m", "sys2_pll_200m", "sys1_pll_266m", +static const char * const imx8mq_pcie2_ctrl_sels[] = {"osc_25m", "sys2_pll_250m", "sys2_pll_200m", "sys1_pll_266m", "sys1_pll_800m", "sys2_pll_500m", "sys2_pll_333m", "sys3_pll2_out", }; -static const char *imx8mq_pcie2_phy_sels[] = {"osc_25m", "sys2_pll_100m", "sys2_pll_500m", "clk_ext1", +static const char * const imx8mq_pcie2_phy_sels[] = {"osc_25m", "sys2_pll_100m", "sys2_pll_500m", "clk_ext1", "clk_ext2", "clk_ext3", "clk_ext4", "sys1_pll_400m", }; -static const char *imx8mq_pcie2_aux_sels[] = {"osc_25m", "sys2_pll_200m", "sys2_pll_50m", "sys3_pll2_out", +static const char * const imx8mq_pcie2_aux_sels[] = {"osc_25m", "sys2_pll_200m", "sys2_pll_50m", "sys3_pll2_out", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_160m", "sys1_pll_200m", }; -static const char *imx8mq_ecspi3_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys1_pll_160m", +static const char * const imx8mq_ecspi3_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys1_pll_160m", "sys1_pll_800m", "sys3_pll2_out", "sys2_pll_250m", "audio_pll2_out", }; -static const char *imx8mq_dram_core_sels[] = {"dram_pll_out", "dram_alt_root", }; +static const char * const imx8mq_dram_core_sels[] = {"dram_pll_out", "dram_alt_root", }; -static const char *imx8mq_clko2_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_400m", "sys2_pll_166m", "audio_pll1_out", - "video_pll1_out", "ckil", }; +static const char * const imx8mq_clko1_sels[] = {"osc_25m", "sys1_pll_800m", "osc_27m", "sys1_pll_200m", + "audio_pll2_out", "sys2_pll_500m", "vpu_pll_out", "sys1_pll_80m", }; +static const char * const imx8mq_clko2_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_400m", "sys2_pll_166m", + "sys3_pll2_out", "audio_pll1_out", "video_pll1_out", "ckil", }; static struct clk_onecell_data clk_data; @@ -308,10 +308,6 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) clks[IMX8MQ_AUDIO_PLL1_REF_DIV] = imx_clk_divider("audio_pll1_ref_div", "audio_pll1_ref_sel", base + 0x0, 5, 6); clks[IMX8MQ_AUDIO_PLL2_REF_DIV] = imx_clk_divider("audio_pll2_ref_div", "audio_pll2_ref_sel", base + 0x8, 5, 6); clks[IMX8MQ_VIDEO_PLL1_REF_DIV] = imx_clk_divider("video_pll1_ref_div", "video_pll1_ref_sel", base + 0x10, 5, 6); - clks[IMX8MQ_SYS1_PLL1_REF_DIV] = imx_clk_divider("sys1_pll1_ref_div", "sys1_pll1_ref_sel", base + 0x38, 25, 3); - clks[IMX8MQ_SYS2_PLL1_REF_DIV] = imx_clk_divider("sys2_pll1_ref_div", "sys2_pll1_ref_sel", base + 0x44, 25, 3); - clks[IMX8MQ_SYS3_PLL1_REF_DIV] = imx_clk_divider("sys3_pll1_ref_div", "sys3_pll1_ref_sel", base + 0x50, 25, 3); - clks[IMX8MQ_DRAM_PLL1_REF_DIV] = imx_clk_divider("dram_pll1_ref_div", "dram_pll1_ref_sel", base + 0x68, 25, 3); clks[IMX8MQ_ARM_PLL] = imx_clk_frac_pll("arm_pll", "arm_pll_ref_div", base + 0x28); clks[IMX8MQ_GPU_PLL] = imx_clk_frac_pll("gpu_pll", "gpu_pll_ref_div", base + 0x18); @@ -319,43 +315,15 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) clks[IMX8MQ_AUDIO_PLL1] = imx_clk_frac_pll("audio_pll1", "audio_pll1_ref_div", base + 0x0); clks[IMX8MQ_AUDIO_PLL2] = imx_clk_frac_pll("audio_pll2", "audio_pll2_ref_div", base + 0x8); clks[IMX8MQ_VIDEO_PLL1] = imx_clk_frac_pll("video_pll1", "video_pll1_ref_div", base + 0x10); - clks[IMX8MQ_SYS1_PLL1] = imx_clk_sccg_pll("sys1_pll1", "sys1_pll1_ref_div", base + 0x30, SCCG_PLL1); - clks[IMX8MQ_SYS2_PLL1] = imx_clk_sccg_pll("sys2_pll1", "sys2_pll1_ref_div", base + 0x3c, SCCG_PLL1); - clks[IMX8MQ_SYS3_PLL1] = imx_clk_sccg_pll("sys3_pll1", "sys3_pll1_ref_div", base + 0x48, SCCG_PLL1); - clks[IMX8MQ_DRAM_PLL1] = imx_clk_sccg_pll("dram_pll1", "dram_pll1_ref_div", base + 0x60, SCCG_PLL1); - - clks[IMX8MQ_SYS1_PLL2] = imx_clk_sccg_pll("sys1_pll2", "sys1_pll1_out_div", base + 0x30, SCCG_PLL2); - clks[IMX8MQ_SYS2_PLL2] = imx_clk_sccg_pll("sys2_pll2", "sys2_pll1_out_div", base + 0x3c, SCCG_PLL2); - clks[IMX8MQ_SYS3_PLL2] = imx_clk_sccg_pll("sys3_pll2", "sys3_pll1_out_div", base + 0x48, SCCG_PLL2); - clks[IMX8MQ_DRAM_PLL2] = imx_clk_sccg_pll("dram_pll2", "dram_pll1_out_div", base + 0x60, SCCG_PLL2); - - /* PLL divs */ - clks[IMX8MQ_SYS1_PLL1_OUT_DIV] = imx_clk_divider("sys1_pll1_out_div", "sys1_pll1_out", base + 0x38, 19, 6); - clks[IMX8MQ_SYS2_PLL1_OUT_DIV] = imx_clk_divider("sys2_pll1_out_div", "sys2_pll1_out", base + 0x44, 19, 6); - clks[IMX8MQ_SYS3_PLL1_OUT_DIV] = imx_clk_divider("sys3_pll1_out_div", "sys3_pll1_out", base + 0x50, 19, 6); - clks[IMX8MQ_DRAM_PLL1_OUT_DIV] = imx_clk_divider("dram_pll1_out_div", "dram_pll1_out", base + 0x68, 19, 6); - clks[IMX8MQ_SYS1_PLL2_DIV] = imx_clk_divider("sys1_pll2_div", "sys1_pll2", base + 0x38, 1, 6); - clks[IMX8MQ_SYS2_PLL2_DIV] = imx_clk_divider("sys2_pll2_div", "sys2_pll2", base + 0x44, 1, 6); - clks[IMX8MQ_SYS3_PLL2_DIV] = imx_clk_divider("sys3_pll2_div", "sys3_pll2", base + 0x50, 1, 6); - clks[IMX8MQ_DRAM_PLL2_DIV] = imx_clk_divider("dram_pll2_div", "dram_pll2", base + 0x68, 1, 6); /* PLL bypass out */ - clks[IMX8MQ_ARM_PLL_BYPASS] = imx_clk_mux("arm_pll_bypass", base + 0x28, 14, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels)); + clks[IMX8MQ_ARM_PLL_BYPASS] = imx_clk_mux_flags("arm_pll_bypass", base + 0x28, 14, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT); clks[IMX8MQ_GPU_PLL_BYPASS] = imx_clk_mux("gpu_pll_bypass", base + 0x18, 14, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels)); clks[IMX8MQ_VPU_PLL_BYPASS] = imx_clk_mux("vpu_pll_bypass", base + 0x20, 14, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels)); clks[IMX8MQ_AUDIO_PLL1_BYPASS] = imx_clk_mux("audio_pll1_bypass", base + 0x0, 14, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels)); clks[IMX8MQ_AUDIO_PLL2_BYPASS] = imx_clk_mux("audio_pll2_bypass", base + 0x8, 14, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels)); clks[IMX8MQ_VIDEO_PLL1_BYPASS] = imx_clk_mux("video_pll1_bypass", base + 0x10, 14, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels)); - clks[IMX8MQ_SYS1_PLL1_OUT] = imx_clk_mux("sys1_pll1_out", base + 0x30, 5, 1, sys1_pll1_out_sels, ARRAY_SIZE(sys1_pll1_out_sels)); - clks[IMX8MQ_SYS2_PLL1_OUT] = imx_clk_mux("sys2_pll1_out", base + 0x3c, 5, 1, sys2_pll1_out_sels, ARRAY_SIZE(sys2_pll1_out_sels)); - clks[IMX8MQ_SYS3_PLL1_OUT] = imx_clk_mux("sys3_pll1_out", base + 0x48, 5, 1, sys3_pll1_out_sels, ARRAY_SIZE(sys3_pll1_out_sels)); - clks[IMX8MQ_DRAM_PLL1_OUT] = imx_clk_mux("dram_pll1_out", base + 0x60, 5, 1, dram_pll1_out_sels, ARRAY_SIZE(dram_pll1_out_sels)); - clks[IMX8MQ_SYS1_PLL2_OUT] = imx_clk_mux("sys1_pll2_out", base + 0x30, 4, 1, sys1_pll2_out_sels, ARRAY_SIZE(sys1_pll2_out_sels)); - clks[IMX8MQ_SYS2_PLL2_OUT] = imx_clk_mux("sys2_pll2_out", base + 0x3c, 4, 1, sys2_pll2_out_sels, ARRAY_SIZE(sys2_pll2_out_sels)); - clks[IMX8MQ_SYS3_PLL2_OUT] = imx_clk_mux("sys3_pll2_out", base + 0x48, 4, 1, sys3_pll2_out_sels, ARRAY_SIZE(sys3_pll2_out_sels)); - clks[IMX8MQ_DRAM_PLL2_OUT] = imx_clk_mux("dram_pll2_out", base + 0x60, 4, 1, dram_pll2_out_sels, ARRAY_SIZE(dram_pll2_out_sels)); - /* PLL OUT GATE */ clks[IMX8MQ_ARM_PLL_OUT] = imx_clk_gate("arm_pll_out", "arm_pll_bypass", base + 0x28, 21); clks[IMX8MQ_GPU_PLL_OUT] = imx_clk_gate("gpu_pll_out", "gpu_pll_bypass", base + 0x18, 21); @@ -363,11 +331,11 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) clks[IMX8MQ_AUDIO_PLL1_OUT] = imx_clk_gate("audio_pll1_out", "audio_pll1_bypass", base + 0x0, 21); clks[IMX8MQ_AUDIO_PLL2_OUT] = imx_clk_gate("audio_pll2_out", "audio_pll2_bypass", base + 0x8, 21); clks[IMX8MQ_VIDEO_PLL1_OUT] = imx_clk_gate("video_pll1_out", "video_pll1_bypass", base + 0x10, 21); - clks[IMX8MQ_SYS1_PLL_OUT] = imx_clk_gate("sys1_pll_out", "sys1_pll2_out", base + 0x30, 9); - clks[IMX8MQ_SYS2_PLL_OUT] = imx_clk_gate("sys2_pll_out", "sys2_pll2_out", base + 0x3c, 9); - clks[IMX8MQ_SYS3_PLL_OUT] = imx_clk_gate("sys3_pll_out", "sys3_pll2_out", base + 0x48, 9); - clks[IMX8MQ_DRAM_PLL_OUT] = imx_clk_gate("dram_pll_out", "dram_pll2_out", base + 0x60, 9); + clks[IMX8MQ_SYS1_PLL_OUT] = imx_clk_sccg_pll("sys1_pll_out", sys1_pll_out_sels, ARRAY_SIZE(sys1_pll_out_sels), 0, 0, 0, base + 0x30, CLK_IS_CRITICAL); + clks[IMX8MQ_SYS2_PLL_OUT] = imx_clk_sccg_pll("sys2_pll_out", sys2_pll_out_sels, ARRAY_SIZE(sys2_pll_out_sels), 0, 0, 1, base + 0x3c, CLK_IS_CRITICAL); + clks[IMX8MQ_SYS3_PLL_OUT] = imx_clk_sccg_pll("sys3_pll_out", sys3_pll_out_sels, ARRAY_SIZE(sys3_pll_out_sels), 0, 0, 1, base + 0x48, CLK_IS_CRITICAL); + clks[IMX8MQ_DRAM_PLL_OUT] = imx_clk_sccg_pll("dram_pll_out", dram_pll_out_sels, ARRAY_SIZE(dram_pll_out_sels), 0, 0, 0, base + 0x60, CLK_IS_CRITICAL); /* SYS PLL fixed output */ clks[IMX8MQ_SYS1_PLL_40M] = imx_clk_fixed_factor("sys1_pll_40m", "sys1_pll_out", 1, 20); clks[IMX8MQ_SYS1_PLL_80M] = imx_clk_fixed_factor("sys1_pll_80m", "sys1_pll_out", 1, 10); @@ -396,15 +364,19 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) /* CORE */ clks[IMX8MQ_CLK_A53_SRC] = imx_clk_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mq_a53_sels, ARRAY_SIZE(imx8mq_a53_sels)); + clks[IMX8MQ_CLK_M4_SRC] = imx_clk_mux2("arm_m4_src", base + 0x8080, 24, 3, imx8mq_arm_m4_sels, ARRAY_SIZE(imx8mq_arm_m4_sels)); clks[IMX8MQ_CLK_VPU_SRC] = imx_clk_mux2("vpu_src", base + 0x8100, 24, 3, imx8mq_vpu_sels, ARRAY_SIZE(imx8mq_vpu_sels)); clks[IMX8MQ_CLK_GPU_CORE_SRC] = imx_clk_mux2("gpu_core_src", base + 0x8180, 24, 3, imx8mq_gpu_core_sels, ARRAY_SIZE(imx8mq_gpu_core_sels)); clks[IMX8MQ_CLK_GPU_SHADER_SRC] = imx_clk_mux2("gpu_shader_src", base + 0x8200, 24, 3, imx8mq_gpu_shader_sels, ARRAY_SIZE(imx8mq_gpu_shader_sels)); + clks[IMX8MQ_CLK_A53_CG] = imx_clk_gate3_flags("arm_a53_cg", "arm_a53_src", base + 0x8000, 28, CLK_IS_CRITICAL); + clks[IMX8MQ_CLK_M4_CG] = imx_clk_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28); clks[IMX8MQ_CLK_VPU_CG] = imx_clk_gate3("vpu_cg", "vpu_src", base + 0x8100, 28); clks[IMX8MQ_CLK_GPU_CORE_CG] = imx_clk_gate3("gpu_core_cg", "gpu_core_src", base + 0x8180, 28); clks[IMX8MQ_CLK_GPU_SHADER_CG] = imx_clk_gate3("gpu_shader_cg", "gpu_shader_src", base + 0x8200, 28); clks[IMX8MQ_CLK_A53_DIV] = imx_clk_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3); + clks[IMX8MQ_CLK_M4_DIV] = imx_clk_divider2("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3); clks[IMX8MQ_CLK_VPU_DIV] = imx_clk_divider2("vpu_div", "vpu_cg", base + 0x8100, 0, 3); clks[IMX8MQ_CLK_GPU_CORE_DIV] = imx_clk_divider2("gpu_core_div", "gpu_core_cg", base + 0x8180, 0, 3); clks[IMX8MQ_CLK_GPU_SHADER_DIV] = imx_clk_divider2("gpu_shader_div", "gpu_shader_cg", base + 0x8200, 0, 3); @@ -479,6 +451,7 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) clks[IMX8MQ_CLK_GPT1] = imx8m_clk_composite("gpt1", imx8mq_gpt1_sels, base + 0xb580); clks[IMX8MQ_CLK_WDOG] = imx8m_clk_composite("wdog", imx8mq_wdog_sels, base + 0xb900); clks[IMX8MQ_CLK_WRCLK] = imx8m_clk_composite("wrclk", imx8mq_wrclk_sels, base + 0xb980); + clks[IMX8MQ_CLK_CLKO1] = imx8m_clk_composite("clko1", imx8mq_clko1_sels, base + 0xba00); clks[IMX8MQ_CLK_CLKO2] = imx8m_clk_composite("clko2", imx8mq_clko2_sels, base + 0xba80); clks[IMX8MQ_CLK_DSI_CORE] = imx8m_clk_composite("dsi_core", imx8mq_dsi_core_sels, base + 0xbb00); clks[IMX8MQ_CLK_DSI_PHY_REF] = imx8m_clk_composite("dsi_phy_ref", imx8mq_dsi_phy_sels, base + 0xbb80); @@ -500,6 +473,11 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) clks[IMX8MQ_CLK_ECSPI2_ROOT] = imx_clk_gate4("ecspi2_root_clk", "ecspi2", base + 0x4080, 0); clks[IMX8MQ_CLK_ECSPI3_ROOT] = imx_clk_gate4("ecspi3_root_clk", "ecspi3", base + 0x4090, 0); clks[IMX8MQ_CLK_ENET1_ROOT] = imx_clk_gate4("enet1_root_clk", "enet_axi", base + 0x40a0, 0); + clks[IMX8MQ_CLK_GPIO1_ROOT] = imx_clk_gate4("gpio1_root_clk", "ipg_root", base + 0x40b0, 0); + clks[IMX8MQ_CLK_GPIO2_ROOT] = imx_clk_gate4("gpio2_root_clk", "ipg_root", base + 0x40c0, 0); + clks[IMX8MQ_CLK_GPIO3_ROOT] = imx_clk_gate4("gpio3_root_clk", "ipg_root", base + 0x40d0, 0); + clks[IMX8MQ_CLK_GPIO4_ROOT] = imx_clk_gate4("gpio4_root_clk", "ipg_root", base + 0x40e0, 0); + clks[IMX8MQ_CLK_GPIO5_ROOT] = imx_clk_gate4("gpio5_root_clk", "ipg_root", base + 0x40f0, 0); clks[IMX8MQ_CLK_GPT1_ROOT] = imx_clk_gate4("gpt1_root_clk", "gpt1", base + 0x4100, 0); clks[IMX8MQ_CLK_I2C1_ROOT] = imx_clk_gate4("i2c1_root_clk", "i2c1", base + 0x4170, 0); clks[IMX8MQ_CLK_I2C2_ROOT] = imx_clk_gate4("i2c2_root_clk", "i2c2", base + 0x4180, 0); @@ -558,6 +536,12 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) clks[IMX8MQ_GPT_3M_CLK] = imx_clk_fixed_factor("gpt_3m", "osc_25m", 1, 8); clks[IMX8MQ_CLK_DRAM_ALT_ROOT] = imx_clk_fixed_factor("dram_alt_root", "dram_alt", 1, 4); + clks[IMX8MQ_CLK_ARM] = imx_clk_cpu("arm", "arm_a53_div", + clks[IMX8MQ_CLK_A53_DIV], + clks[IMX8MQ_CLK_A53_SRC], + clks[IMX8MQ_ARM_PLL_OUT], + clks[IMX8MQ_SYS1_PLL_800M]); + for (i = 0; i < IMX8MQ_CLK_END; i++) if (IS_ERR(clks[i])) pr_err("i.MX8mq clk %u register failed with %ld\n", diff --git a/drivers/clk/imx/clk-imx8qxp.c b/drivers/clk/imx/clk-imx8qxp.c index 83e2ef96d81da..5e2903efc4881 100644 --- a/drivers/clk/imx/clk-imx8qxp.c +++ b/drivers/clk/imx/clk-imx8qxp.c @@ -138,6 +138,7 @@ static int imx8qxp_clk_probe(struct platform_device *pdev) } static const struct of_device_id imx8qxp_match[] = { + { .compatible = "fsl,scu-clk", }, { .compatible = "fsl,imx8qxp-clk", }, { /* sentinel */ } }; diff --git a/drivers/clk/imx/clk-pll14xx.c b/drivers/clk/imx/clk-pll14xx.c new file mode 100644 index 0000000000000..1acfa3e3cfb40 --- /dev/null +++ b/drivers/clk/imx/clk-pll14xx.c @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2017-2018 NXP. + */ + +#include <linux/bitops.h> +#include <linux/clk-provider.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/slab.h> +#include <linux/jiffies.h> + +#include "clk.h" + +#define GNRL_CTL 0x0 +#define DIV_CTL 0x4 +#define LOCK_STATUS BIT(31) +#define LOCK_SEL_MASK BIT(29) +#define CLKE_MASK BIT(11) +#define RST_MASK BIT(9) +#define BYPASS_MASK BIT(4) +#define MDIV_SHIFT 12 +#define MDIV_MASK GENMASK(21, 12) +#define PDIV_SHIFT 4 +#define PDIV_MASK GENMASK(9, 4) +#define SDIV_SHIFT 0 +#define SDIV_MASK GENMASK(2, 0) +#define KDIV_SHIFT 0 +#define KDIV_MASK GENMASK(15, 0) + +#define LOCK_TIMEOUT_US 10000 + +struct clk_pll14xx { + struct clk_hw hw; + void __iomem *base; + enum imx_pll14xx_type type; + const struct imx_pll14xx_rate_table *rate_table; + int rate_count; +}; + +#define to_clk_pll14xx(_hw) container_of(_hw, struct clk_pll14xx, hw) + +static const struct imx_pll14xx_rate_table *imx_get_pll_settings( + struct clk_pll14xx *pll, unsigned long rate) +{ + const struct imx_pll14xx_rate_table *rate_table = pll->rate_table; + int i; + + for (i = 0; i < pll->rate_count; i++) + if (rate == rate_table[i].rate) + return &rate_table[i]; + + return NULL; +} + +static long clk_pll14xx_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *prate) +{ + struct clk_pll14xx *pll = to_clk_pll14xx(hw); + const struct imx_pll14xx_rate_table *rate_table = pll->rate_table; + int i; + + /* Assumming rate_table is in descending order */ + for (i = 0; i < pll->rate_count; i++) + if (rate >= rate_table[i].rate) + return rate_table[i].rate; + + /* return minimum supported value */ + return rate_table[i - 1].rate; +} + +static unsigned long clk_pll1416x_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct clk_pll14xx *pll = to_clk_pll14xx(hw); + u32 mdiv, pdiv, sdiv, pll_gnrl, pll_div; + u64 fvco = parent_rate; + + pll_gnrl = readl_relaxed(pll->base); + pll_div = readl_relaxed(pll->base + 4); + mdiv = (pll_div & MDIV_MASK) >> MDIV_SHIFT; + pdiv = (pll_div & PDIV_MASK) >> PDIV_SHIFT; + sdiv = (pll_div & SDIV_MASK) >> SDIV_SHIFT; + + fvco *= mdiv; + do_div(fvco, pdiv << sdiv); + + return fvco; +} + +static unsigned long clk_pll1443x_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct clk_pll14xx *pll = to_clk_pll14xx(hw); + u32 mdiv, pdiv, sdiv, pll_gnrl, pll_div_ctl0, pll_div_ctl1; + short int kdiv; + u64 fvco = parent_rate; + + pll_gnrl = readl_relaxed(pll->base); + pll_div_ctl0 = readl_relaxed(pll->base + 4); + pll_div_ctl1 = readl_relaxed(pll->base + 8); + mdiv = (pll_div_ctl0 & MDIV_MASK) >> MDIV_SHIFT; + pdiv = (pll_div_ctl0 & PDIV_MASK) >> PDIV_SHIFT; + sdiv = (pll_div_ctl0 & SDIV_MASK) >> SDIV_SHIFT; + kdiv = pll_div_ctl1 & KDIV_MASK; + + /* fvco = (m * 65536 + k) * Fin / (p * 65536) */ + fvco *= (mdiv * 65536 + kdiv); + pdiv *= 65536; + + do_div(fvco, pdiv << sdiv); + + return fvco; +} + +static inline bool clk_pll1416x_mp_change(const struct imx_pll14xx_rate_table *rate, + u32 pll_div) +{ + u32 old_mdiv, old_pdiv; + + old_mdiv = (pll_div >> MDIV_SHIFT) & MDIV_MASK; + old_pdiv = (pll_div >> PDIV_SHIFT) & PDIV_MASK; + + return rate->mdiv != old_mdiv || rate->pdiv != old_pdiv; +} + +static inline bool clk_pll1443x_mpk_change(const struct imx_pll14xx_rate_table *rate, + u32 pll_div_ctl0, u32 pll_div_ctl1) +{ + u32 old_mdiv, old_pdiv, old_kdiv; + + old_mdiv = (pll_div_ctl0 >> MDIV_SHIFT) & MDIV_MASK; + old_pdiv = (pll_div_ctl0 >> PDIV_SHIFT) & PDIV_MASK; + old_kdiv = (pll_div_ctl1 >> KDIV_SHIFT) & KDIV_MASK; + + return rate->mdiv != old_mdiv || rate->pdiv != old_pdiv || + rate->kdiv != old_kdiv; +} + +static inline bool clk_pll1443x_mp_change(const struct imx_pll14xx_rate_table *rate, + u32 pll_div_ctl0, u32 pll_div_ctl1) +{ + u32 old_mdiv, old_pdiv, old_kdiv; + + old_mdiv = (pll_div_ctl0 >> MDIV_SHIFT) & MDIV_MASK; + old_pdiv = (pll_div_ctl0 >> PDIV_SHIFT) & PDIV_MASK; + old_kdiv = (pll_div_ctl1 >> KDIV_SHIFT) & KDIV_MASK; + + return rate->mdiv != old_mdiv || rate->pdiv != old_pdiv || + rate->kdiv != old_kdiv; +} + +static int clk_pll14xx_wait_lock(struct clk_pll14xx *pll) +{ + u32 val; + + return readl_poll_timeout(pll->base, val, val & LOCK_TIMEOUT_US, 0, + LOCK_TIMEOUT_US); +} + +static int clk_pll1416x_set_rate(struct clk_hw *hw, unsigned long drate, + unsigned long prate) +{ + struct clk_pll14xx *pll = to_clk_pll14xx(hw); + const struct imx_pll14xx_rate_table *rate; + u32 tmp, div_val; + int ret; + + rate = imx_get_pll_settings(pll, drate); + if (!rate) { + pr_err("%s: Invalid rate : %lu for pll clk %s\n", __func__, + drate, clk_hw_get_name(hw)); + return -EINVAL; + } + + tmp = readl_relaxed(pll->base + 4); + + if (!clk_pll1416x_mp_change(rate, tmp)) { + tmp &= ~(SDIV_MASK) << SDIV_SHIFT; + tmp |= rate->sdiv << SDIV_SHIFT; + writel_relaxed(tmp, pll->base + 4); + + return 0; + } + + /* Bypass clock and set lock to pll output lock */ + tmp = readl_relaxed(pll->base); + tmp |= LOCK_SEL_MASK; + writel_relaxed(tmp, pll->base); + + /* Enable RST */ + tmp &= ~RST_MASK; + writel_relaxed(tmp, pll->base); + + div_val = (rate->mdiv << MDIV_SHIFT) | (rate->pdiv << PDIV_SHIFT) | + (rate->sdiv << SDIV_SHIFT); + writel_relaxed(div_val, pll->base + 0x4); + + /* + * According to SPEC, t3 - t2 need to be greater than + * 1us and 1/FREF, respectively. + * FREF is FIN / Prediv, the prediv is [1, 63], so choose + * 3us. + */ + udelay(3); + + /* Disable RST */ + tmp |= RST_MASK; + writel_relaxed(tmp, pll->base); + + /* Wait Lock */ + ret = clk_pll14xx_wait_lock(pll); + if (ret) + return ret; + + /* Bypass */ + tmp &= ~BYPASS_MASK; + writel_relaxed(tmp, pll->base); + + return 0; +} + +static int clk_pll1443x_set_rate(struct clk_hw *hw, unsigned long drate, + unsigned long prate) +{ + struct clk_pll14xx *pll = to_clk_pll14xx(hw); + const struct imx_pll14xx_rate_table *rate; + u32 tmp, div_val; + int ret; + + rate = imx_get_pll_settings(pll, drate); + if (!rate) { + pr_err("%s: Invalid rate : %lu for pll clk %s\n", __func__, + drate, clk_hw_get_name(hw)); + return -EINVAL; + } + + tmp = readl_relaxed(pll->base + 4); + div_val = readl_relaxed(pll->base + 8); + + if (!clk_pll1443x_mpk_change(rate, tmp, div_val)) { + tmp &= ~(SDIV_MASK) << SDIV_SHIFT; + tmp |= rate->sdiv << SDIV_SHIFT; + writel_relaxed(tmp, pll->base + 4); + + return 0; + } + + /* Enable RST */ + tmp = readl_relaxed(pll->base); + tmp &= ~RST_MASK; + writel_relaxed(tmp, pll->base); + + div_val = (rate->mdiv << MDIV_SHIFT) | (rate->pdiv << PDIV_SHIFT) | + (rate->sdiv << SDIV_SHIFT); + writel_relaxed(div_val, pll->base + 0x4); + writel_relaxed(rate->kdiv << KDIV_SHIFT, pll->base + 0x8); + + /* + * According to SPEC, t3 - t2 need to be greater than + * 1us and 1/FREF, respectively. + * FREF is FIN / Prediv, the prediv is [1, 63], so choose + * 3us. + */ + udelay(3); + + /* Disable RST */ + tmp |= RST_MASK; + writel_relaxed(tmp, pll->base); + + /* Wait Lock*/ + ret = clk_pll14xx_wait_lock(pll); + if (ret) + return ret; + + /* Bypass */ + tmp &= ~BYPASS_MASK; + writel_relaxed(tmp, pll->base); + + return 0; +} + +static int clk_pll14xx_prepare(struct clk_hw *hw) +{ + struct clk_pll14xx *pll = to_clk_pll14xx(hw); + u32 val; + + /* + * RESETB = 1 from 0, PLL starts its normal + * operation after lock time + */ + val = readl_relaxed(pll->base + GNRL_CTL); + val |= RST_MASK; + writel_relaxed(val, pll->base + GNRL_CTL); + + return clk_pll14xx_wait_lock(pll); +} + +static int clk_pll14xx_is_prepared(struct clk_hw *hw) +{ + struct clk_pll14xx *pll = to_clk_pll14xx(hw); + u32 val; + + val = readl_relaxed(pll->base + GNRL_CTL); + + return (val & RST_MASK) ? 1 : 0; +} + +static void clk_pll14xx_unprepare(struct clk_hw *hw) +{ + struct clk_pll14xx *pll = to_clk_pll14xx(hw); + u32 val; + + /* + * Set RST to 0, power down mode is enabled and + * every digital block is reset + */ + val = readl_relaxed(pll->base + GNRL_CTL); + val &= ~RST_MASK; + writel_relaxed(val, pll->base + GNRL_CTL); +} + +static const struct clk_ops clk_pll1416x_ops = { + .prepare = clk_pll14xx_prepare, + .unprepare = clk_pll14xx_unprepare, + .is_prepared = clk_pll14xx_is_prepared, + .recalc_rate = clk_pll1416x_recalc_rate, + .round_rate = clk_pll14xx_round_rate, + .set_rate = clk_pll1416x_set_rate, +}; + +static const struct clk_ops clk_pll1416x_min_ops = { + .recalc_rate = clk_pll1416x_recalc_rate, +}; + +static const struct clk_ops clk_pll1443x_ops = { + .prepare = clk_pll14xx_prepare, + .unprepare = clk_pll14xx_unprepare, + .is_prepared = clk_pll14xx_is_prepared, + .recalc_rate = clk_pll1443x_recalc_rate, + .round_rate = clk_pll14xx_round_rate, + .set_rate = clk_pll1443x_set_rate, +}; + +struct clk *imx_clk_pll14xx(const char *name, const char *parent_name, + void __iomem *base, + const struct imx_pll14xx_clk *pll_clk) +{ + struct clk_pll14xx *pll; + struct clk *clk; + struct clk_init_data init; + + pll = kzalloc(sizeof(*pll), GFP_KERNEL); + if (!pll) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.flags = pll_clk->flags; + init.parent_names = &parent_name; + init.num_parents = 1; + + switch (pll_clk->type) { + case PLL_1416X: + if (!pll->rate_table) + init.ops = &clk_pll1416x_min_ops; + else + init.ops = &clk_pll1416x_ops; + break; + case PLL_1443X: + init.ops = &clk_pll1443x_ops; + break; + default: + pr_err("%s: Unknown pll type for pll clk %s\n", + __func__, name); + }; + + pll->base = base; + pll->hw.init = &init; + pll->type = pll_clk->type; + pll->rate_table = pll_clk->rate_table; + pll->rate_count = pll_clk->rate_count; + + clk = clk_register(NULL, &pll->hw); + if (IS_ERR(clk)) { + pr_err("%s: failed to register pll %s %lu\n", + __func__, name, PTR_ERR(clk)); + kfree(pll); + } + + return clk; +} diff --git a/drivers/clk/imx/clk-sccg-pll.c b/drivers/clk/imx/clk-sccg-pll.c index ee7752bace895..9dfd03a955574 100644 --- a/drivers/clk/imx/clk-sccg-pll.c +++ b/drivers/clk/imx/clk-sccg-pll.c @@ -25,87 +25,292 @@ #define PLL_DIVF2_MASK GENMASK(12, 7) #define PLL_DIVR1_MASK GENMASK(27, 25) #define PLL_DIVR2_MASK GENMASK(24, 19) +#define PLL_DIVQ_MASK GENMASK(6, 1) #define PLL_REF_MASK GENMASK(2, 0) #define PLL_LOCK_MASK BIT(31) #define PLL_PD_MASK BIT(7) -#define OSC_25M 25000000 -#define OSC_27M 27000000 +/* These are the specification limits for the SSCG PLL */ +#define PLL_REF_MIN_FREQ 25000000UL +#define PLL_REF_MAX_FREQ 235000000UL -#define PLL_SCCG_LOCK_TIMEOUT 70 +#define PLL_STAGE1_MIN_FREQ 1600000000UL +#define PLL_STAGE1_MAX_FREQ 2400000000UL + +#define PLL_STAGE1_REF_MIN_FREQ 25000000UL +#define PLL_STAGE1_REF_MAX_FREQ 54000000UL + +#define PLL_STAGE2_MIN_FREQ 1200000000UL +#define PLL_STAGE2_MAX_FREQ 2400000000UL + +#define PLL_STAGE2_REF_MIN_FREQ 54000000UL +#define PLL_STAGE2_REF_MAX_FREQ 75000000UL + +#define PLL_OUT_MIN_FREQ 20000000UL +#define PLL_OUT_MAX_FREQ 1200000000UL + +#define PLL_DIVR1_MAX 7 +#define PLL_DIVR2_MAX 63 +#define PLL_DIVF1_MAX 63 +#define PLL_DIVF2_MAX 63 +#define PLL_DIVQ_MAX 63 + +#define PLL_BYPASS_NONE 0x0 +#define PLL_BYPASS1 0x2 +#define PLL_BYPASS2 0x1 + +#define SSCG_PLL_BYPASS1_MASK BIT(5) +#define SSCG_PLL_BYPASS2_MASK BIT(4) +#define SSCG_PLL_BYPASS_MASK GENMASK(5, 4) + +#define PLL_SCCG_LOCK_TIMEOUT 70 + +struct clk_sccg_pll_setup { + int divr1, divf1; + int divr2, divf2; + int divq; + int bypass; + + uint64_t vco1; + uint64_t vco2; + uint64_t fout; + uint64_t ref; + uint64_t ref_div1; + uint64_t ref_div2; + uint64_t fout_request; + int fout_error; +}; struct clk_sccg_pll { struct clk_hw hw; - void __iomem *base; + const struct clk_ops ops; + + void __iomem *base; + + struct clk_sccg_pll_setup setup; + + u8 parent; + u8 bypass1; + u8 bypass2; }; #define to_clk_sccg_pll(_hw) container_of(_hw, struct clk_sccg_pll, hw) -static int clk_pll_wait_lock(struct clk_sccg_pll *pll) +static int clk_sccg_pll_wait_lock(struct clk_sccg_pll *pll) { u32 val; - return readl_poll_timeout(pll->base, val, val & PLL_LOCK_MASK, 0, - PLL_SCCG_LOCK_TIMEOUT); + val = readl_relaxed(pll->base + PLL_CFG0); + + /* don't wait for lock if all plls are bypassed */ + if (!(val & SSCG_PLL_BYPASS2_MASK)) + return readl_poll_timeout(pll->base, val, val & PLL_LOCK_MASK, + 0, PLL_SCCG_LOCK_TIMEOUT); + + return 0; } -static int clk_pll1_is_prepared(struct clk_hw *hw) +static int clk_sccg_pll2_check_match(struct clk_sccg_pll_setup *setup, + struct clk_sccg_pll_setup *temp_setup) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); - u32 val; + int new_diff = temp_setup->fout - temp_setup->fout_request; + int diff = temp_setup->fout_error; - val = readl_relaxed(pll->base + PLL_CFG0); - return (val & PLL_PD_MASK) ? 0 : 1; + if (abs(diff) > abs(new_diff)) { + temp_setup->fout_error = new_diff; + memcpy(setup, temp_setup, sizeof(struct clk_sccg_pll_setup)); + + if (temp_setup->fout_request == temp_setup->fout) + return 0; + } + return -1; } -static unsigned long clk_pll1_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) +static int clk_sccg_divq_lookup(struct clk_sccg_pll_setup *setup, + struct clk_sccg_pll_setup *temp_setup) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); - u32 val, divf; + int ret = -EINVAL; + + for (temp_setup->divq = 0; temp_setup->divq <= PLL_DIVQ_MAX; + temp_setup->divq++) { + temp_setup->vco2 = temp_setup->vco1; + do_div(temp_setup->vco2, temp_setup->divr2 + 1); + temp_setup->vco2 *= 2; + temp_setup->vco2 *= temp_setup->divf2 + 1; + if (temp_setup->vco2 >= PLL_STAGE2_MIN_FREQ && + temp_setup->vco2 <= PLL_STAGE2_MAX_FREQ) { + temp_setup->fout = temp_setup->vco2; + do_div(temp_setup->fout, 2 * (temp_setup->divq + 1)); + + ret = clk_sccg_pll2_check_match(setup, temp_setup); + if (!ret) { + temp_setup->bypass = PLL_BYPASS1; + return ret; + } + } + } - val = readl_relaxed(pll->base + PLL_CFG2); - divf = FIELD_GET(PLL_DIVF1_MASK, val); + return ret; +} + +static int clk_sccg_divf2_lookup(struct clk_sccg_pll_setup *setup, + struct clk_sccg_pll_setup *temp_setup) +{ + int ret = -EINVAL; + + for (temp_setup->divf2 = 0; temp_setup->divf2 <= PLL_DIVF2_MAX; + temp_setup->divf2++) { + ret = clk_sccg_divq_lookup(setup, temp_setup); + if (!ret) + return ret; + } - return parent_rate * 2 * (divf + 1); + return ret; } -static long clk_pll1_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) +static int clk_sccg_divr2_lookup(struct clk_sccg_pll_setup *setup, + struct clk_sccg_pll_setup *temp_setup) { - unsigned long parent_rate = *prate; - u32 div; + int ret = -EINVAL; + + for (temp_setup->divr2 = 0; temp_setup->divr2 <= PLL_DIVR2_MAX; + temp_setup->divr2++) { + temp_setup->ref_div2 = temp_setup->vco1; + do_div(temp_setup->ref_div2, temp_setup->divr2 + 1); + if (temp_setup->ref_div2 >= PLL_STAGE2_REF_MIN_FREQ && + temp_setup->ref_div2 <= PLL_STAGE2_REF_MAX_FREQ) { + ret = clk_sccg_divf2_lookup(setup, temp_setup); + if (!ret) + return ret; + } + } + + return ret; +} + +static int clk_sccg_pll2_find_setup(struct clk_sccg_pll_setup *setup, + struct clk_sccg_pll_setup *temp_setup, + uint64_t ref) +{ + + int ret = -EINVAL; - if (!parent_rate) - return 0; + if (ref < PLL_STAGE1_MIN_FREQ || ref > PLL_STAGE1_MAX_FREQ) + return ret; - div = rate / (parent_rate * 2); + temp_setup->vco1 = ref; - return parent_rate * div * 2; + ret = clk_sccg_divr2_lookup(setup, temp_setup); + return ret; } -static int clk_pll1_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) +static int clk_sccg_divf1_lookup(struct clk_sccg_pll_setup *setup, + struct clk_sccg_pll_setup *temp_setup) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); - u32 val; - u32 divf; + int ret = -EINVAL; - if (!parent_rate) - return -EINVAL; + for (temp_setup->divf1 = 0; temp_setup->divf1 <= PLL_DIVF1_MAX; + temp_setup->divf1++) { + uint64_t vco1 = temp_setup->ref; - divf = rate / (parent_rate * 2); + do_div(vco1, temp_setup->divr1 + 1); + vco1 *= 2; + vco1 *= temp_setup->divf1 + 1; - val = readl_relaxed(pll->base + PLL_CFG2); - val &= ~PLL_DIVF1_MASK; - val |= FIELD_PREP(PLL_DIVF1_MASK, divf - 1); - writel_relaxed(val, pll->base + PLL_CFG2); + ret = clk_sccg_pll2_find_setup(setup, temp_setup, vco1); + if (!ret) { + temp_setup->bypass = PLL_BYPASS_NONE; + return ret; + } + } + + return ret; +} + +static int clk_sccg_divr1_lookup(struct clk_sccg_pll_setup *setup, + struct clk_sccg_pll_setup *temp_setup) +{ + int ret = -EINVAL; + + for (temp_setup->divr1 = 0; temp_setup->divr1 <= PLL_DIVR1_MAX; + temp_setup->divr1++) { + temp_setup->ref_div1 = temp_setup->ref; + do_div(temp_setup->ref_div1, temp_setup->divr1 + 1); + if (temp_setup->ref_div1 >= PLL_STAGE1_REF_MIN_FREQ && + temp_setup->ref_div1 <= PLL_STAGE1_REF_MAX_FREQ) { + ret = clk_sccg_divf1_lookup(setup, temp_setup); + if (!ret) + return ret; + } + } + + return ret; +} + +static int clk_sccg_pll1_find_setup(struct clk_sccg_pll_setup *setup, + struct clk_sccg_pll_setup *temp_setup, + uint64_t ref) +{ + + int ret = -EINVAL; + + if (ref < PLL_REF_MIN_FREQ || ref > PLL_REF_MAX_FREQ) + return ret; + + temp_setup->ref = ref; + + ret = clk_sccg_divr1_lookup(setup, temp_setup); + + return ret; +} + +static int clk_sccg_pll_find_setup(struct clk_sccg_pll_setup *setup, + uint64_t prate, + uint64_t rate, int try_bypass) +{ + struct clk_sccg_pll_setup temp_setup; + int ret = -EINVAL; + + memset(&temp_setup, 0, sizeof(struct clk_sccg_pll_setup)); + memset(setup, 0, sizeof(struct clk_sccg_pll_setup)); + + temp_setup.fout_error = PLL_OUT_MAX_FREQ; + temp_setup.fout_request = rate; + + switch (try_bypass) { - return clk_pll_wait_lock(pll); + case PLL_BYPASS2: + if (prate == rate) { + setup->bypass = PLL_BYPASS2; + setup->fout = rate; + ret = 0; + } + break; + + case PLL_BYPASS1: + ret = clk_sccg_pll2_find_setup(setup, &temp_setup, prate); + break; + + case PLL_BYPASS_NONE: + ret = clk_sccg_pll1_find_setup(setup, &temp_setup, prate); + break; + } + + return ret; +} + + +static int clk_sccg_pll_is_prepared(struct clk_hw *hw) +{ + struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); + + u32 val = readl_relaxed(pll->base + PLL_CFG0); + + return (val & PLL_PD_MASK) ? 0 : 1; } -static int clk_pll1_prepare(struct clk_hw *hw) +static int clk_sccg_pll_prepare(struct clk_hw *hw) { struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); u32 val; @@ -114,10 +319,10 @@ static int clk_pll1_prepare(struct clk_hw *hw) val &= ~PLL_PD_MASK; writel_relaxed(val, pll->base + PLL_CFG0); - return clk_pll_wait_lock(pll); + return clk_sccg_pll_wait_lock(pll); } -static void clk_pll1_unprepare(struct clk_hw *hw) +static void clk_sccg_pll_unprepare(struct clk_hw *hw) { struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); u32 val; @@ -125,121 +330,208 @@ static void clk_pll1_unprepare(struct clk_hw *hw) val = readl_relaxed(pll->base + PLL_CFG0); val |= PLL_PD_MASK; writel_relaxed(val, pll->base + PLL_CFG0); - } -static unsigned long clk_pll2_recalc_rate(struct clk_hw *hw, +static unsigned long clk_sccg_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); - u32 val, ref, divr1, divf1, divr2, divf2; + u32 val, divr1, divf1, divr2, divf2, divq; u64 temp64; - val = readl_relaxed(pll->base + PLL_CFG0); - switch (FIELD_GET(PLL_REF_MASK, val)) { - case 0: - ref = OSC_25M; - break; - case 1: - ref = OSC_27M; - break; - default: - ref = OSC_25M; - break; - } - val = readl_relaxed(pll->base + PLL_CFG2); divr1 = FIELD_GET(PLL_DIVR1_MASK, val); divr2 = FIELD_GET(PLL_DIVR2_MASK, val); divf1 = FIELD_GET(PLL_DIVF1_MASK, val); divf2 = FIELD_GET(PLL_DIVF2_MASK, val); - - temp64 = ref * 2; - temp64 *= (divf1 + 1) * (divf2 + 1); - - do_div(temp64, (divr1 + 1) * (divr2 + 1)); + divq = FIELD_GET(PLL_DIVQ_MASK, val); + + temp64 = parent_rate; + + val = clk_readl(pll->base + PLL_CFG0); + if (val & SSCG_PLL_BYPASS2_MASK) { + temp64 = parent_rate; + } else if (val & SSCG_PLL_BYPASS1_MASK) { + temp64 *= divf2; + do_div(temp64, (divr2 + 1) * (divq + 1)); + } else { + temp64 *= 2; + temp64 *= (divf1 + 1) * (divf2 + 1); + do_div(temp64, (divr1 + 1) * (divr2 + 1) * (divq + 1)); + } return temp64; } -static long clk_pll2_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) +static int clk_sccg_pll_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) { - u32 div; - unsigned long parent_rate = *prate; + struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); + struct clk_sccg_pll_setup *setup = &pll->setup; + u32 val; - if (!parent_rate) - return 0; + /* set bypass here too since the parent might be the same */ + val = clk_readl(pll->base + PLL_CFG0); + val &= ~SSCG_PLL_BYPASS_MASK; + val |= FIELD_PREP(SSCG_PLL_BYPASS_MASK, setup->bypass); + clk_writel(val, pll->base + PLL_CFG0); - div = rate / parent_rate; + val = readl_relaxed(pll->base + PLL_CFG2); + val &= ~(PLL_DIVF1_MASK | PLL_DIVF2_MASK); + val &= ~(PLL_DIVR1_MASK | PLL_DIVR2_MASK | PLL_DIVQ_MASK); + val |= FIELD_PREP(PLL_DIVF1_MASK, setup->divf1); + val |= FIELD_PREP(PLL_DIVF2_MASK, setup->divf2); + val |= FIELD_PREP(PLL_DIVR1_MASK, setup->divr1); + val |= FIELD_PREP(PLL_DIVR2_MASK, setup->divr2); + val |= FIELD_PREP(PLL_DIVQ_MASK, setup->divq); + writel_relaxed(val, pll->base + PLL_CFG2); - return parent_rate * div; + return clk_sccg_pll_wait_lock(pll); } -static int clk_pll2_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) +static u8 clk_sccg_pll_get_parent(struct clk_hw *hw) { + struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); u32 val; - u32 divf; + u8 ret = pll->parent; + + val = clk_readl(pll->base + PLL_CFG0); + if (val & SSCG_PLL_BYPASS2_MASK) + ret = pll->bypass2; + else if (val & SSCG_PLL_BYPASS1_MASK) + ret = pll->bypass1; + return ret; +} + +static int clk_sccg_pll_set_parent(struct clk_hw *hw, u8 index) +{ struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); + u32 val; - if (!parent_rate) - return -EINVAL; + val = clk_readl(pll->base + PLL_CFG0); + val &= ~SSCG_PLL_BYPASS_MASK; + val |= FIELD_PREP(SSCG_PLL_BYPASS_MASK, pll->setup.bypass); + clk_writel(val, pll->base + PLL_CFG0); - divf = rate / parent_rate; + return clk_sccg_pll_wait_lock(pll); +} - val = readl_relaxed(pll->base + PLL_CFG2); - val &= ~PLL_DIVF2_MASK; - val |= FIELD_PREP(PLL_DIVF2_MASK, divf - 1); - writel_relaxed(val, pll->base + PLL_CFG2); +static int __clk_sccg_pll_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req, + uint64_t min, + uint64_t max, + uint64_t rate, + int bypass) +{ + struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); + struct clk_sccg_pll_setup *setup = &pll->setup; + struct clk_hw *parent_hw = NULL; + int bypass_parent_index; + int ret = -EINVAL; + + req->max_rate = max; + req->min_rate = min; + + switch (bypass) { + case PLL_BYPASS2: + bypass_parent_index = pll->bypass2; + break; + case PLL_BYPASS1: + bypass_parent_index = pll->bypass1; + break; + default: + bypass_parent_index = pll->parent; + break; + } + + parent_hw = clk_hw_get_parent_by_index(hw, bypass_parent_index); + ret = __clk_determine_rate(parent_hw, req); + if (!ret) { + ret = clk_sccg_pll_find_setup(setup, req->rate, + rate, bypass); + } + + req->best_parent_hw = parent_hw; + req->best_parent_rate = req->rate; + req->rate = setup->fout; - return clk_pll_wait_lock(pll); + return ret; } -static const struct clk_ops clk_sccg_pll1_ops = { - .is_prepared = clk_pll1_is_prepared, - .recalc_rate = clk_pll1_recalc_rate, - .round_rate = clk_pll1_round_rate, - .set_rate = clk_pll1_set_rate, -}; +static int clk_sccg_pll_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); + struct clk_sccg_pll_setup *setup = &pll->setup; + uint64_t rate = req->rate; + uint64_t min = req->min_rate; + uint64_t max = req->max_rate; + int ret = -EINVAL; + + if (rate < PLL_OUT_MIN_FREQ || rate > PLL_OUT_MAX_FREQ) + return ret; + + ret = __clk_sccg_pll_determine_rate(hw, req, req->rate, req->rate, + rate, PLL_BYPASS2); + if (!ret) + return ret; + + ret = __clk_sccg_pll_determine_rate(hw, req, PLL_STAGE1_REF_MIN_FREQ, + PLL_STAGE1_REF_MAX_FREQ, rate, + PLL_BYPASS1); + if (!ret) + return ret; + + ret = __clk_sccg_pll_determine_rate(hw, req, PLL_REF_MIN_FREQ, + PLL_REF_MAX_FREQ, rate, + PLL_BYPASS_NONE); + if (!ret) + return ret; + + if (setup->fout >= min && setup->fout <= max) + ret = 0; + + return ret; +} -static const struct clk_ops clk_sccg_pll2_ops = { - .prepare = clk_pll1_prepare, - .unprepare = clk_pll1_unprepare, - .recalc_rate = clk_pll2_recalc_rate, - .round_rate = clk_pll2_round_rate, - .set_rate = clk_pll2_set_rate, +static const struct clk_ops clk_sccg_pll_ops = { + .prepare = clk_sccg_pll_prepare, + .unprepare = clk_sccg_pll_unprepare, + .is_prepared = clk_sccg_pll_is_prepared, + .recalc_rate = clk_sccg_pll_recalc_rate, + .set_rate = clk_sccg_pll_set_rate, + .set_parent = clk_sccg_pll_set_parent, + .get_parent = clk_sccg_pll_get_parent, + .determine_rate = clk_sccg_pll_determine_rate, }; struct clk *imx_clk_sccg_pll(const char *name, - const char *parent_name, + const char * const *parent_names, + u8 num_parents, + u8 parent, u8 bypass1, u8 bypass2, void __iomem *base, - enum imx_sccg_pll_type pll_type) + unsigned long flags) { struct clk_sccg_pll *pll; struct clk_init_data init; struct clk_hw *hw; int ret; - switch (pll_type) { - case SCCG_PLL1: - init.ops = &clk_sccg_pll1_ops; - break; - case SCCG_PLL2: - init.ops = &clk_sccg_pll2_ops; - break; - default: - return ERR_PTR(-EINVAL); - } - pll = kzalloc(sizeof(*pll), GFP_KERNEL); if (!pll) return ERR_PTR(-ENOMEM); + pll->parent = parent; + pll->bypass1 = bypass1; + pll->bypass2 = bypass2; + + pll->base = base; init.name = name; - init.flags = 0; - init.parent_names = &parent_name; - init.num_parents = 1; + init.ops = &clk_sccg_pll_ops; + + init.flags = flags; + init.parent_names = parent_names; + init.num_parents = num_parents; pll->base = base; pll->hw.init = &init; diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c index 7ccf7edfe11cb..fbef740704d0c 100644 --- a/drivers/clk/imx/clk-scu.c +++ b/drivers/clk/imx/clk-scu.c @@ -4,12 +4,17 @@ * Dong Aisheng <aisheng.dong@nxp.com> */ +#include <dt-bindings/firmware/imx/rsrc.h> +#include <linux/arm-smccc.h> #include <linux/clk-provider.h> #include <linux/err.h> #include <linux/slab.h> #include "clk-scu.h" +#define IMX_SIP_CPUFREQ 0xC2000001 +#define IMX_SIP_SET_CPUFREQ 0x00 + static struct imx_sc_ipc *ccm_ipc_handle; /* @@ -66,6 +71,41 @@ struct imx_sc_msg_get_clock_rate { }; /* + * struct imx_sc_msg_get_clock_parent - clock get parent protocol + * @hdr: SCU protocol header + * @req: get parent request protocol + * @resp: get parent response protocol + * + * This structure describes the SCU protocol of clock get parent + */ +struct imx_sc_msg_get_clock_parent { + struct imx_sc_rpc_msg hdr; + union { + struct req_get_clock_parent { + __le16 resource; + u8 clk; + } __packed req; + struct resp_get_clock_parent { + u8 parent; + } resp; + } data; +}; + +/* + * struct imx_sc_msg_set_clock_parent - clock set parent protocol + * @hdr: SCU protocol header + * @req: set parent request protocol + * + * This structure describes the SCU protocol of clock set parent + */ +struct imx_sc_msg_set_clock_parent { + struct imx_sc_rpc_msg hdr; + __le16 resource; + u8 clk; + u8 parent; +} __packed; + +/* * struct imx_sc_msg_req_clock_enable - clock gate protocol * @hdr: SCU protocol header * @resource: clock resource to gate @@ -145,6 +185,25 @@ static long clk_scu_round_rate(struct clk_hw *hw, unsigned long rate, return rate; } +static int clk_scu_atf_set_cpu_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_scu *clk = to_clk_scu(hw); + struct arm_smccc_res res; + unsigned long cluster_id; + + if (clk->rsrc_id == IMX_SC_R_A35) + cluster_id = 0; + else + return -EINVAL; + + /* CPU frequency scaling can ONLY be done by ARM-Trusted-Firmware */ + arm_smccc_smc(IMX_SIP_CPUFREQ, IMX_SIP_SET_CPUFREQ, + cluster_id, rate, 0, 0, 0, 0, &res); + + return 0; +} + /* * clk_scu_set_rate - Set rate for a SCU clock * @hw: clock to change rate for @@ -173,6 +232,49 @@ static int clk_scu_set_rate(struct clk_hw *hw, unsigned long rate, return imx_scu_call_rpc(ccm_ipc_handle, &msg, true); } +static u8 clk_scu_get_parent(struct clk_hw *hw) +{ + struct clk_scu *clk = to_clk_scu(hw); + struct imx_sc_msg_get_clock_parent msg; + struct imx_sc_rpc_msg *hdr = &msg.hdr; + int ret; + + hdr->ver = IMX_SC_RPC_VERSION; + hdr->svc = IMX_SC_RPC_SVC_PM; + hdr->func = IMX_SC_PM_FUNC_GET_CLOCK_PARENT; + hdr->size = 2; + + msg.data.req.resource = cpu_to_le16(clk->rsrc_id); + msg.data.req.clk = clk->clk_type; + + ret = imx_scu_call_rpc(ccm_ipc_handle, &msg, true); + if (ret) { + pr_err("%s: failed to get clock parent %d\n", + clk_hw_get_name(hw), ret); + return 0; + } + + return msg.data.resp.parent; +} + +static int clk_scu_set_parent(struct clk_hw *hw, u8 index) +{ + struct clk_scu *clk = to_clk_scu(hw); + struct imx_sc_msg_set_clock_parent msg; + struct imx_sc_rpc_msg *hdr = &msg.hdr; + + hdr->ver = IMX_SC_RPC_VERSION; + hdr->svc = IMX_SC_RPC_SVC_PM; + hdr->func = IMX_SC_PM_FUNC_SET_CLOCK_PARENT; + hdr->size = 2; + + msg.resource = cpu_to_le16(clk->rsrc_id); + msg.clk = clk->clk_type; + msg.parent = index; + + return imx_scu_call_rpc(ccm_ipc_handle, &msg, true); +} + static int sc_pm_clock_enable(struct imx_sc_ipc *ipc, u16 resource, u8 clk, bool enable, bool autog) { @@ -228,11 +330,22 @@ static const struct clk_ops clk_scu_ops = { .recalc_rate = clk_scu_recalc_rate, .round_rate = clk_scu_round_rate, .set_rate = clk_scu_set_rate, + .get_parent = clk_scu_get_parent, + .set_parent = clk_scu_set_parent, + .prepare = clk_scu_prepare, + .unprepare = clk_scu_unprepare, +}; + +static const struct clk_ops clk_scu_cpu_ops = { + .recalc_rate = clk_scu_recalc_rate, + .round_rate = clk_scu_round_rate, + .set_rate = clk_scu_atf_set_cpu_rate, .prepare = clk_scu_prepare, .unprepare = clk_scu_unprepare, }; -struct clk_hw *imx_clk_scu(const char *name, u32 rsrc_id, u8 clk_type) +struct clk_hw *__imx_clk_scu(const char *name, const char * const *parents, + int num_parents, u32 rsrc_id, u8 clk_type) { struct clk_init_data init; struct clk_scu *clk; @@ -248,7 +361,13 @@ struct clk_hw *imx_clk_scu(const char *name, u32 rsrc_id, u8 clk_type) init.name = name; init.ops = &clk_scu_ops; - init.num_parents = 0; + if (rsrc_id == IMX_SC_R_A35) + init.ops = &clk_scu_cpu_ops; + else + init.ops = &clk_scu_ops; + init.parent_names = parents; + init.num_parents = num_parents; + /* * Note on MX8, the clocks are tightly coupled with power domain * that once the power domain is off, the clock status may be diff --git a/drivers/clk/imx/clk-scu.h b/drivers/clk/imx/clk-scu.h index 52c1746ec9887..2bcfaf06a4586 100644 --- a/drivers/clk/imx/clk-scu.h +++ b/drivers/clk/imx/clk-scu.h @@ -10,7 +10,21 @@ #include <linux/firmware/imx/sci.h> int imx_clk_scu_init(void); -struct clk_hw *imx_clk_scu(const char *name, u32 rsrc_id, u8 clk_type); + +struct clk_hw *__imx_clk_scu(const char *name, const char * const *parents, + int num_parents, u32 rsrc_id, u8 clk_type); + +static inline struct clk_hw *imx_clk_scu(const char *name, u32 rsrc_id, + u8 clk_type) +{ + return __imx_clk_scu(name, NULL, 0, rsrc_id, clk_type); +} + +static inline struct clk_hw *imx_clk_scu2(const char *name, const char * const *parents, + int num_parents, u32 rsrc_id, u8 clk_type) +{ + return __imx_clk_scu(name, parents, num_parents, rsrc_id, clk_type); +} struct clk_hw *imx_clk_lpcg_scu(const char *name, const char *parent_name, unsigned long flags, void __iomem *reg, diff --git a/drivers/clk/imx/clk-vf610.c b/drivers/clk/imx/clk-vf610.c index 6dae54325a91d..a334667c450a1 100644 --- a/drivers/clk/imx/clk-vf610.c +++ b/drivers/clk/imx/clk-vf610.c @@ -203,6 +203,7 @@ static void __init vf610_clocks_init(struct device_node *ccm_node) np = of_find_compatible_node(NULL, NULL, "fsl,vf610-anatop"); anatop_base = of_iomap(np, 0); BUG_ON(!anatop_base); + of_node_put(np); np = ccm_node; ccm_base = of_iomap(np, 0); diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h index 028312de21b81..5748ec8673e45 100644 --- a/drivers/clk/imx/clk.h +++ b/drivers/clk/imx/clk.h @@ -27,6 +27,30 @@ enum imx_sccg_pll_type { SCCG_PLL2, }; +enum imx_pll14xx_type { + PLL_1416X, + PLL_1443X, +}; + +/* NOTE: Rate table should be kept sorted in descending order. */ +struct imx_pll14xx_rate_table { + unsigned int rate; + unsigned int pdiv; + unsigned int mdiv; + unsigned int sdiv; + unsigned int kdiv; +}; + +struct imx_pll14xx_clk { + enum imx_pll14xx_type type; + const struct imx_pll14xx_rate_table *rate_table; + int rate_count; + int flags; +}; + +struct clk *imx_clk_pll14xx(const char *name, const char *parent_name, + void __iomem *base, const struct imx_pll14xx_clk *pll_clk); + struct clk *imx_clk_pllv1(enum imx_pllv1_type type, const char *name, const char *parent, void __iomem *base); @@ -36,9 +60,12 @@ struct clk *imx_clk_pllv2(const char *name, const char *parent, struct clk *imx_clk_frac_pll(const char *name, const char *parent_name, void __iomem *base); -struct clk *imx_clk_sccg_pll(const char *name, const char *parent_name, - void __iomem *base, - enum imx_sccg_pll_type pll_type); +struct clk *imx_clk_sccg_pll(const char *name, + const char * const *parent_names, + u8 num_parents, + u8 parent, u8 bypass1, u8 bypass2, + void __iomem *base, + unsigned long flags); enum imx_pllv3_type { IMX_PLLV3_GENERIC, @@ -329,7 +356,8 @@ static inline struct clk *imx_clk_mux_flags(const char *name, } static inline struct clk *imx_clk_mux2_flags(const char *name, - void __iomem *reg, u8 shift, u8 width, const char **parents, + void __iomem *reg, u8 shift, u8 width, + const char * const *parents, int num_parents, unsigned long flags) { return clk_register_mux(NULL, name, parents, num_parents, @@ -354,7 +382,7 @@ struct clk *imx_clk_cpu(const char *name, const char *parent_name, struct clk *step); struct clk *imx8m_clk_composite_flags(const char *name, - const char **parent_names, + const char * const *parent_names, int num_parents, void __iomem *reg, unsigned long flags); diff --git a/drivers/clk/ingenic/cgu.c b/drivers/clk/ingenic/cgu.c index 5ef7d9ba2195d..510b685212d3f 100644 --- a/drivers/clk/ingenic/cgu.c +++ b/drivers/clk/ingenic/cgu.c @@ -83,7 +83,7 @@ ingenic_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) const struct ingenic_cgu_clk_info *clk_info; const struct ingenic_cgu_pll_info *pll_info; unsigned m, n, od_enc, od; - bool bypass, enable; + bool bypass; unsigned long flags; u32 ctl; @@ -103,7 +103,6 @@ ingenic_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) od_enc &= GENMASK(pll_info->od_bits - 1, 0); bypass = !pll_info->no_bypass_bit && !!(ctl & BIT(pll_info->bypass_bit)); - enable = !!(ctl & BIT(pll_info->enable_bit)); if (bypass) return parent_rate; @@ -426,16 +425,16 @@ ingenic_clk_round_rate(struct clk_hw *hw, unsigned long req_rate, struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw); struct ingenic_cgu *cgu = ingenic_clk->cgu; const struct ingenic_cgu_clk_info *clk_info; - long rate = *parent_rate; + unsigned int div = 1; clk_info = &cgu->clock_info[ingenic_clk->idx]; if (clk_info->type & CGU_CLK_DIV) - rate /= ingenic_clk_calc_div(clk_info, *parent_rate, req_rate); + div = ingenic_clk_calc_div(clk_info, *parent_rate, req_rate); else if (clk_info->type & CGU_CLK_FIXDIV) - rate /= clk_info->fixdiv.div; + div = clk_info->fixdiv.div; - return rate; + return DIV_ROUND_UP(*parent_rate, div); } static int @@ -455,7 +454,7 @@ ingenic_clk_set_rate(struct clk_hw *hw, unsigned long req_rate, if (clk_info->type & CGU_CLK_DIV) { div = ingenic_clk_calc_div(clk_info, parent_rate, req_rate); - rate = parent_rate / div; + rate = DIV_ROUND_UP(parent_rate, div); if (rate != req_rate) return -EINVAL; diff --git a/drivers/clk/ingenic/cgu.h b/drivers/clk/ingenic/cgu.h index 502bcbb61b047..e12716d8ce3cf 100644 --- a/drivers/clk/ingenic/cgu.h +++ b/drivers/clk/ingenic/cgu.h @@ -80,7 +80,7 @@ struct ingenic_cgu_mux_info { * @reg: offset of the divider control register within the CGU * @shift: number of bits to left shift the divide value by (ie. the index of * the lowest bit of the divide value within its control register) - * @div: number of bits to divide the divider value by (i.e. if the + * @div: number to divide the divider value by (i.e. if the * effective divider value is the value written to the register * multiplied by some constant) * @bits: the size of the divide value in bits diff --git a/drivers/clk/ingenic/jz4740-cgu.c b/drivers/clk/ingenic/jz4740-cgu.c index 4479c102e8994..b86edd3282493 100644 --- a/drivers/clk/ingenic/jz4740-cgu.c +++ b/drivers/clk/ingenic/jz4740-cgu.c @@ -165,7 +165,7 @@ static const struct ingenic_cgu_clk_info jz4740_cgu_clocks[] = { .parents = { JZ4740_CLK_EXT, JZ4740_CLK_PLL_HALF, -1, -1 }, .mux = { CGU_REG_CPCCR, 29, 1 }, .div = { CGU_REG_CPCCR, 23, 1, 6, -1, -1, -1 }, - .gate = { CGU_REG_SCR, 6 }, + .gate = { CGU_REG_SCR, 6, true }, }, /* Gate-only clocks */ diff --git a/drivers/clk/mediatek/clk-gate.c b/drivers/clk/mediatek/clk-gate.c index 934bf0e45e262..9628d4e7690bb 100644 --- a/drivers/clk/mediatek/clk-gate.c +++ b/drivers/clk/mediatek/clk-gate.c @@ -157,7 +157,8 @@ struct clk *mtk_clk_register_gate( int clr_ofs, int sta_ofs, u8 bit, - const struct clk_ops *ops) + const struct clk_ops *ops, + unsigned long flags) { struct mtk_clk_gate *cg; struct clk *clk; @@ -172,6 +173,7 @@ struct clk *mtk_clk_register_gate( init.parent_names = parent_name ? &parent_name : NULL; init.num_parents = parent_name ? 1 : 0; init.ops = ops; + init.flags = flags; cg->regmap = regmap; cg->set_ofs = set_ofs; diff --git a/drivers/clk/mediatek/clk-gate.h b/drivers/clk/mediatek/clk-gate.h index 72ef89b3ad7ba..9f766dfe1d573 100644 --- a/drivers/clk/mediatek/clk-gate.h +++ b/drivers/clk/mediatek/clk-gate.h @@ -47,6 +47,7 @@ struct clk *mtk_clk_register_gate( int clr_ofs, int sta_ofs, u8 bit, - const struct clk_ops *ops); + const struct clk_ops *ops, + unsigned long flags); #endif /* __DRV_CLK_GATE_H */ diff --git a/drivers/clk/mediatek/clk-mt2701.c b/drivers/clk/mediatek/clk-mt2701.c index ab6ab07f53e64..905a2316f6a7f 100644 --- a/drivers/clk/mediatek/clk-mt2701.c +++ b/drivers/clk/mediatek/clk-mt2701.c @@ -535,8 +535,8 @@ static const struct mtk_composite top_muxes[] = { 0x0080, 8, 2, 15), MUX_GATE(CLK_TOP_DPI0_SEL, "dpi0_sel", dpi0_parents, 0x0080, 16, 3, 23), - MUX_GATE(CLK_TOP_DPI1_SEL, "dpi1_sel", dpi1_parents, - 0x0080, 24, 2, 31), + MUX_GATE_FLAGS_2(CLK_TOP_DPI1_SEL, "dpi1_sel", dpi1_parents, + 0x0080, 24, 2, 31, 0, CLK_MUX_ROUND_CLOSEST), MUX_GATE(CLK_TOP_TVE_SEL, "tve_sel", tve_parents, 0x0090, 0, 3, 7), diff --git a/drivers/clk/mediatek/clk-mt2712.c b/drivers/clk/mediatek/clk-mt2712.c index 991d4093726e5..b09cb3d99f66f 100644 --- a/drivers/clk/mediatek/clk-mt2712.c +++ b/drivers/clk/mediatek/clk-mt2712.c @@ -223,6 +223,8 @@ static const struct mtk_fixed_factor top_divs[] = { 4), FACTOR(CLK_TOP_APLL1_D3, "apll1_d3", "apll1_ck", 1, 3), + FACTOR(CLK_TOP_APLL2_D3, "apll2_d3", "apll2_ck", 1, + 3), }; static const char * const axi_parents[] = { @@ -594,7 +596,8 @@ static const char * const a1sys_hp_parents[] = { "apll1_ck", "apll1_d2", "apll1_d4", - "apll1_d8" + "apll1_d8", + "apll1_d3" }; static const char * const a2sys_hp_parents[] = { @@ -602,7 +605,8 @@ static const char * const a2sys_hp_parents[] = { "apll2_ck", "apll2_d2", "apll2_d4", - "apll2_d8" + "apll2_d8", + "apll2_d3" }; static const char * const asm_l_parents[] = { @@ -1463,7 +1467,6 @@ static struct platform_driver clk_mt2712_drv = { .probe = clk_mt2712_probe, .driver = { .name = "clk-mt2712", - .owner = THIS_MODULE, .of_match_table = of_match_clk_mt2712, }, }; diff --git a/drivers/clk/mediatek/clk-mt6797.c b/drivers/clk/mediatek/clk-mt6797.c index 5702bc974ed99..c2b46b184b9a4 100644 --- a/drivers/clk/mediatek/clk-mt6797.c +++ b/drivers/clk/mediatek/clk-mt6797.c @@ -324,6 +324,10 @@ static const char * const anc_md32_parents[] = { "univpll_d5", }; +/* + * Clock mux ddrphycfg is needed by the DRAM controller. We mark it as + * critical as otherwise the system will hang after boot. + */ static const struct mtk_composite top_muxes[] = { MUX(CLK_TOP_MUX_ULPOSC_AXI_CK_MUX_PRE, "ulposc_axi_ck_mux_pre", ulposc_axi_ck_mux_pre_parents, 0x0040, 3, 1), @@ -331,8 +335,8 @@ static const struct mtk_composite top_muxes[] = { ulposc_axi_ck_mux_parents, 0x0040, 2, 1), MUX(CLK_TOP_MUX_AXI, "axi_sel", axi_parents, 0x0040, 0, 2), - MUX(CLK_TOP_MUX_DDRPHYCFG, "ddrphycfg_sel", ddrphycfg_parents, - 0x0040, 16, 2), + MUX_FLAGS(CLK_TOP_MUX_DDRPHYCFG, "ddrphycfg_sel", ddrphycfg_parents, + 0x0040, 16, 2, CLK_IS_CRITICAL | CLK_SET_RATE_PARENT), MUX(CLK_TOP_MUX_MM, "mm_sel", mm_parents, 0x0040, 24, 2), MUX_GATE(CLK_TOP_MUX_PWM, "pwm_sel", pwm_parents, 0x0050, 0, 3, 7), @@ -424,33 +428,45 @@ static const struct mtk_gate_regs infra2_cg_regs = { .sta_ofs = 0x00b0, }; -#define GATE_ICG0(_id, _name, _parent, _shift) { \ - .id = _id, \ - .name = _name, \ - .parent_name = _parent, \ - .regs = &infra0_cg_regs, \ - .shift = _shift, \ - .ops = &mtk_clk_gate_ops_setclr, \ +#define GATE_ICG0(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &infra0_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ } -#define GATE_ICG1(_id, _name, _parent, _shift) { \ - .id = _id, \ - .name = _name, \ - .parent_name = _parent, \ - .regs = &infra1_cg_regs, \ - .shift = _shift, \ - .ops = &mtk_clk_gate_ops_setclr, \ +#define GATE_ICG1(_id, _name, _parent, _shift) \ + GATE_ICG1_FLAGS(_id, _name, _parent, _shift, 0) + +#define GATE_ICG1_FLAGS(_id, _name, _parent, _shift, _flags) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &infra1_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + .flags = _flags, \ } -#define GATE_ICG2(_id, _name, _parent, _shift) { \ - .id = _id, \ - .name = _name, \ - .parent_name = _parent, \ - .regs = &infra2_cg_regs, \ - .shift = _shift, \ - .ops = &mtk_clk_gate_ops_setclr, \ +#define GATE_ICG2(_id, _name, _parent, _shift) \ + GATE_ICG2_FLAGS(_id, _name, _parent, _shift, 0) + +#define GATE_ICG2_FLAGS(_id, _name, _parent, _shift, _flags) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &infra2_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + .flags = _flags, \ } +/* + * Clock gates dramc and dramc_b are needed by the DRAM controller. + * We mark them as critical as otherwise the system will hang after boot. + */ static const struct mtk_gate infra_clks[] = { GATE_ICG0(CLK_INFRA_PMIC_TMR, "infra_pmic_tmr", "ulposc", 0), GATE_ICG0(CLK_INFRA_PMIC_AP, "infra_pmic_ap", "pmicspi_sel", 1), @@ -505,7 +521,8 @@ static const struct mtk_gate infra_clks[] = { GATE_ICG1(CLK_INFRA_CCIF_AP, "infra_ccif_ap", "axi_sel", 23), GATE_ICG1(CLK_INFRA_AUDIO, "infra_audio", "axi_sel", 25), GATE_ICG1(CLK_INFRA_CCIF_MD, "infra_ccif_md", "axi_sel", 26), - GATE_ICG1(CLK_INFRA_DRAMC_F26M, "infra_dramc_f26m", "clk26m", 31), + GATE_ICG1_FLAGS(CLK_INFRA_DRAMC_F26M, "infra_dramc_f26m", + "clk26m", 31, CLK_IS_CRITICAL), GATE_ICG2(CLK_INFRA_I2C4, "infra_i2c4", "axi_sel", 0), GATE_ICG2(CLK_INFRA_I2C_APPM, "infra_i2c_appm", "axi_sel", 1), GATE_ICG2(CLK_INFRA_I2C_GPUPM, "infra_i2c_gpupm", "axi_sel", 2), @@ -516,7 +533,8 @@ static const struct mtk_gate infra_clks[] = { GATE_ICG2(CLK_INFRA_I2C5, "infra_i2c5", "axi_sel", 7), GATE_ICG2(CLK_INFRA_SYS_CIRQ, "infra_sys_cirq", "axi_sel", 8), GATE_ICG2(CLK_INFRA_SPI1, "infra_spi1", "spi_sel", 10), - GATE_ICG2(CLK_INFRA_DRAMC_B_F26M, "infra_dramc_b_f26m", "clk26m", 11), + GATE_ICG2_FLAGS(CLK_INFRA_DRAMC_B_F26M, "infra_dramc_b_f26m", + "clk26m", 11, CLK_IS_CRITICAL), GATE_ICG2(CLK_INFRA_ANC_MD32, "infra_anc_md32", "anc_md32_sel", 12), GATE_ICG2(CLK_INFRA_ANC_MD32_32K, "infra_anc_md32_32k", "clk26m", 13), GATE_ICG2(CLK_INFRA_DVFS_SPM1, "infra_dvfs_spm1", "axi_sel", 15), diff --git a/drivers/clk/mediatek/clk-mt8173.c b/drivers/clk/mediatek/clk-mt8173.c index 96c292c3e440a..deedeb3ea33b5 100644 --- a/drivers/clk/mediatek/clk-mt8173.c +++ b/drivers/clk/mediatek/clk-mt8173.c @@ -533,7 +533,7 @@ static const char * const ca53_parents[] __initconst = { "univpll" }; -static const char * const ca57_parents[] __initconst = { +static const char * const ca72_parents[] __initconst = { "clk26m", "armca15pll", "mainpll", @@ -542,7 +542,7 @@ static const char * const ca57_parents[] __initconst = { static const struct mtk_composite cpu_muxes[] __initconst = { MUX(CLK_INFRA_CA53SEL, "infra_ca53_sel", ca53_parents, 0x0000, 0, 2), - MUX(CLK_INFRA_CA57SEL, "infra_ca57_sel", ca57_parents, 0x0000, 2, 2), + MUX(CLK_INFRA_CA72SEL, "infra_ca72_sel", ca72_parents, 0x0000, 2, 2), }; static const struct mtk_composite top_muxes[] __initconst = { diff --git a/drivers/clk/mediatek/clk-mtk.c b/drivers/clk/mediatek/clk-mtk.c index 9c0ae4278a946..5531dd2e496d6 100644 --- a/drivers/clk/mediatek/clk-mtk.c +++ b/drivers/clk/mediatek/clk-mtk.c @@ -130,7 +130,7 @@ int mtk_clk_register_gates(struct device_node *node, gate->regs->set_ofs, gate->regs->clr_ofs, gate->regs->sta_ofs, - gate->shift, gate->ops); + gate->shift, gate->ops, gate->flags); if (IS_ERR(clk)) { pr_err("Failed to register clk %s: %ld\n", @@ -167,7 +167,7 @@ struct clk *mtk_clk_register_composite(const struct mtk_composite *mc, mux->mask = BIT(mc->mux_width) - 1; mux->shift = mc->mux_shift; mux->lock = lock; - + mux->flags = mc->mux_flags; mux_hw = &mux->hw; mux_ops = &clk_mux_ops; diff --git a/drivers/clk/mediatek/clk-mtk.h b/drivers/clk/mediatek/clk-mtk.h index f83c2bbb677ea..fb27b5bf30d95 100644 --- a/drivers/clk/mediatek/clk-mtk.h +++ b/drivers/clk/mediatek/clk-mtk.h @@ -81,15 +81,13 @@ struct mtk_composite { signed char divider_shift; signed char divider_width; + u8 mux_flags; + signed char num_parents; }; -/* - * In case the rate change propagation to parent clocks is undesirable, - * this macro allows to specify the clock flags manually. - */ -#define MUX_GATE_FLAGS(_id, _name, _parents, _reg, _shift, _width, \ - _gate, _flags) { \ +#define MUX_GATE_FLAGS_2(_id, _name, _parents, _reg, _shift, \ + _width, _gate, _flags, _muxflags) { \ .id = _id, \ .name = _name, \ .mux_reg = _reg, \ @@ -101,9 +99,19 @@ struct mtk_composite { .parent_names = _parents, \ .num_parents = ARRAY_SIZE(_parents), \ .flags = _flags, \ + .mux_flags = _muxflags, \ } /* + * In case the rate change propagation to parent clocks is undesirable, + * this macro allows to specify the clock flags manually. + */ +#define MUX_GATE_FLAGS(_id, _name, _parents, _reg, _shift, _width, \ + _gate, _flags) \ + MUX_GATE_FLAGS_2(_id, _name, _parents, _reg, \ + _shift, _width, _gate, _flags, 0) + +/* * Unless necessary, all MUX_GATE clocks propagate rate changes to their * parent clock by default. */ @@ -111,7 +119,11 @@ struct mtk_composite { MUX_GATE_FLAGS(_id, _name, _parents, _reg, _shift, _width, \ _gate, CLK_SET_RATE_PARENT) -#define MUX(_id, _name, _parents, _reg, _shift, _width) { \ +#define MUX(_id, _name, _parents, _reg, _shift, _width) \ + MUX_FLAGS(_id, _name, _parents, _reg, \ + _shift, _width, CLK_SET_RATE_PARENT) + +#define MUX_FLAGS(_id, _name, _parents, _reg, _shift, _width, _flags) { \ .id = _id, \ .name = _name, \ .mux_reg = _reg, \ @@ -121,7 +133,7 @@ struct mtk_composite { .divider_shift = -1, \ .parent_names = _parents, \ .num_parents = ARRAY_SIZE(_parents), \ - .flags = CLK_SET_RATE_PARENT, \ + .flags = _flags, \ } #define DIV_GATE(_id, _name, _parent, _gate_reg, _gate_shift, _div_reg, \ @@ -158,6 +170,7 @@ struct mtk_gate { const struct mtk_gate_regs *regs; int shift; const struct clk_ops *ops; + unsigned long flags; }; int mtk_clk_register_gates(struct device_node *node, diff --git a/drivers/clk/meson/Kconfig b/drivers/clk/meson/Kconfig index efaa70f682b45..3858747f54388 100644 --- a/drivers/clk/meson/Kconfig +++ b/drivers/clk/meson/Kconfig @@ -1,27 +1,52 @@ -config COMMON_CLK_AMLOGIC - bool - depends on ARCH_MESON || COMPILE_TEST - select COMMON_CLK_REGMAP_MESON +config COMMON_CLK_MESON_INPUT + tristate -config COMMON_CLK_AMLOGIC_AUDIO - bool - depends on ARCH_MESON || COMPILE_TEST - select COMMON_CLK_AMLOGIC +config COMMON_CLK_MESON_REGMAP + tristate + select REGMAP -config COMMON_CLK_MESON_AO - bool - depends on OF - depends on ARCH_MESON || COMPILE_TEST - select COMMON_CLK_REGMAP_MESON +config COMMON_CLK_MESON_DUALDIV + tristate + select COMMON_CLK_MESON_REGMAP + +config COMMON_CLK_MESON_MPLL + tristate + select COMMON_CLK_MESON_REGMAP + +config COMMON_CLK_MESON_PHASE + tristate + select COMMON_CLK_MESON_REGMAP + +config COMMON_CLK_MESON_PLL + tristate + select COMMON_CLK_MESON_REGMAP + +config COMMON_CLK_MESON_SCLK_DIV + tristate + select COMMON_CLK_MESON_REGMAP + +config COMMON_CLK_MESON_VID_PLL_DIV + tristate + select COMMON_CLK_MESON_REGMAP + +config COMMON_CLK_MESON_AO_CLKC + tristate + select COMMON_CLK_MESON_REGMAP + select COMMON_CLK_MESON_INPUT select RESET_CONTROLLER -config COMMON_CLK_REGMAP_MESON - bool - select REGMAP +config COMMON_CLK_MESON_EE_CLKC + tristate + select COMMON_CLK_MESON_REGMAP + select COMMON_CLK_MESON_INPUT config COMMON_CLK_MESON8B bool - select COMMON_CLK_AMLOGIC + depends on ARCH_MESON + select COMMON_CLK_MESON_REGMAP + select COMMON_CLK_MESON_MPLL + select COMMON_CLK_MESON_PLL + select MFD_SYSCON select RESET_CONTROLLER help Support for the clock controller on AmLogic S802 (Meson8), @@ -30,8 +55,14 @@ config COMMON_CLK_MESON8B config COMMON_CLK_GXBB bool - select COMMON_CLK_AMLOGIC - select COMMON_CLK_MESON_AO + depends on ARCH_MESON + select COMMON_CLK_MESON_REGMAP + select COMMON_CLK_MESON_DUALDIV + select COMMON_CLK_MESON_VID_PLL_DIV + select COMMON_CLK_MESON_MPLL + select COMMON_CLK_MESON_PLL + select COMMON_CLK_MESON_AO_CLKC + select COMMON_CLK_MESON_EE_CLKC select MFD_SYSCON help Support for the clock controller on AmLogic S905 devices, aka gxbb. @@ -39,8 +70,13 @@ config COMMON_CLK_GXBB config COMMON_CLK_AXG bool - select COMMON_CLK_AMLOGIC - select COMMON_CLK_MESON_AO + depends on ARCH_MESON + select COMMON_CLK_MESON_REGMAP + select COMMON_CLK_MESON_DUALDIV + select COMMON_CLK_MESON_MPLL + select COMMON_CLK_MESON_PLL + select COMMON_CLK_MESON_AO_CLKC + select COMMON_CLK_MESON_EE_CLKC select MFD_SYSCON help Support for the clock controller on AmLogic A113D devices, aka axg. @@ -48,9 +84,26 @@ config COMMON_CLK_AXG config COMMON_CLK_AXG_AUDIO tristate "Meson AXG Audio Clock Controller Driver" - depends on COMMON_CLK_AXG - select COMMON_CLK_AMLOGIC_AUDIO - select MFD_SYSCON + depends on ARCH_MESON + select COMMON_CLK_MESON_INPUT + select COMMON_CLK_MESON_REGMAP + select COMMON_CLK_MESON_PHASE + select COMMON_CLK_MESON_SCLK_DIV + select REGMAP_MMIO help Support for the audio clock controller on AmLogic A113D devices, aka axg, Say Y if you want audio subsystem to work. + +config COMMON_CLK_G12A + bool + depends on ARCH_MESON + select COMMON_CLK_MESON_REGMAP + select COMMON_CLK_MESON_DUALDIV + select COMMON_CLK_MESON_MPLL + select COMMON_CLK_MESON_PLL + select COMMON_CLK_MESON_AO_CLKC + select COMMON_CLK_MESON_EE_CLKC + select MFD_SYSCON + help + Support for the clock controller on Amlogic S905D2, S905X2 and S905Y2 + devices, aka g12a. Say Y if you want peripherals to work. diff --git a/drivers/clk/meson/Makefile b/drivers/clk/meson/Makefile index a849aa809825a..021fc290e749b 100644 --- a/drivers/clk/meson/Makefile +++ b/drivers/clk/meson/Makefile @@ -1,13 +1,20 @@ -# -# Makefile for Meson specific clk -# +# Amlogic clock drivers -obj-$(CONFIG_COMMON_CLK_AMLOGIC) += clk-pll.o clk-mpll.o clk-phase.o vid-pll-div.o -obj-$(CONFIG_COMMON_CLK_AMLOGIC) += clk-input.o -obj-$(CONFIG_COMMON_CLK_AMLOGIC_AUDIO) += clk-triphase.o sclk-div.o -obj-$(CONFIG_COMMON_CLK_MESON_AO) += meson-aoclk.o +obj-$(CONFIG_COMMON_CLK_MESON_AO_CLKC) += meson-aoclk.o +obj-$(CONFIG_COMMON_CLK_MESON_DUALDIV) += clk-dualdiv.o +obj-$(CONFIG_COMMON_CLK_MESON_EE_CLKC) += meson-eeclk.o +obj-$(CONFIG_COMMON_CLK_MESON_INPUT) += clk-input.o +obj-$(CONFIG_COMMON_CLK_MESON_MPLL) += clk-mpll.o +obj-$(CONFIG_COMMON_CLK_MESON_PHASE) += clk-phase.o +obj-$(CONFIG_COMMON_CLK_MESON_PLL) += clk-pll.o +obj-$(CONFIG_COMMON_CLK_MESON_REGMAP) += clk-regmap.o +obj-$(CONFIG_COMMON_CLK_MESON_SCLK_DIV) += sclk-div.o +obj-$(CONFIG_COMMON_CLK_MESON_VID_PLL_DIV) += vid-pll-div.o + +# Amlogic Clock controllers + +obj-$(CONFIG_COMMON_CLK_AXG) += axg.o axg-aoclk.o +obj-$(CONFIG_COMMON_CLK_AXG_AUDIO) += axg-audio.o +obj-$(CONFIG_COMMON_CLK_GXBB) += gxbb.o gxbb-aoclk.o +obj-$(CONFIG_COMMON_CLK_G12A) += g12a.o g12a-aoclk.o obj-$(CONFIG_COMMON_CLK_MESON8B) += meson8b.o -obj-$(CONFIG_COMMON_CLK_GXBB) += gxbb.o gxbb-aoclk.o gxbb-aoclk-32k.o -obj-$(CONFIG_COMMON_CLK_AXG) += axg.o axg-aoclk.o -obj-$(CONFIG_COMMON_CLK_AXG_AUDIO) += axg-audio.o -obj-$(CONFIG_COMMON_CLK_REGMAP_MESON) += clk-regmap.o diff --git a/drivers/clk/meson/axg-aoclk.c b/drivers/clk/meson/axg-aoclk.c index 29e0885423871..0086f31288eb8 100644 --- a/drivers/clk/meson/axg-aoclk.c +++ b/drivers/clk/meson/axg-aoclk.c @@ -12,10 +12,27 @@ #include <linux/platform_device.h> #include <linux/reset-controller.h> #include <linux/mfd/syscon.h> -#include "clk-regmap.h" #include "meson-aoclk.h" #include "axg-aoclk.h" +#include "clk-regmap.h" +#include "clk-dualdiv.h" + +#define IN_PREFIX "ao-in-" + +/* + * AO Configuration Clock registers offsets + * Register offsets from the data sheet must be multiplied by 4. + */ +#define AO_RTI_PWR_CNTL_REG1 0x0C +#define AO_RTI_PWR_CNTL_REG0 0x10 +#define AO_RTI_GEN_CNTL_REG0 0x40 +#define AO_OSCIN_CNTL 0x58 +#define AO_CRT_CLK_CNTL1 0x68 +#define AO_SAR_CLK 0x90 +#define AO_RTC_ALT_CLK_CNTL0 0x94 +#define AO_RTC_ALT_CLK_CNTL1 0x98 + #define AXG_AO_GATE(_name, _bit) \ static struct clk_regmap axg_aoclk_##_name = { \ .data = &(struct clk_regmap_gate_data) { \ @@ -25,7 +42,7 @@ static struct clk_regmap axg_aoclk_##_name = { \ .hw.init = &(struct clk_init_data) { \ .name = "axg_ao_" #_name, \ .ops = &clk_regmap_gate_ops, \ - .parent_names = (const char *[]){ "clk81" }, \ + .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk" }, \ .num_parents = 1, \ .flags = CLK_IGNORE_UNUSED, \ }, \ @@ -39,17 +56,141 @@ AXG_AO_GATE(uart2, 5); AXG_AO_GATE(ir_blaster, 6); AXG_AO_GATE(saradc, 7); +static struct clk_regmap axg_aoclk_cts_oscin = { + .data = &(struct clk_regmap_gate_data){ + .offset = AO_RTI_PWR_CNTL_REG0, + .bit_idx = 14, + }, + .hw.init = &(struct clk_init_data){ + .name = "cts_oscin", + .ops = &clk_regmap_gate_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap axg_aoclk_32k_pre = { + .data = &(struct clk_regmap_gate_data){ + .offset = AO_RTC_ALT_CLK_CNTL0, + .bit_idx = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "axg_ao_32k_pre", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "cts_oscin" }, + .num_parents = 1, + }, +}; + +static const struct meson_clk_dualdiv_param axg_32k_div_table[] = { + { + .dual = 1, + .n1 = 733, + .m1 = 8, + .n2 = 732, + .m2 = 11, + }, {} +}; + +static struct clk_regmap axg_aoclk_32k_div = { + .data = &(struct meson_clk_dualdiv_data){ + .n1 = { + .reg_off = AO_RTC_ALT_CLK_CNTL0, + .shift = 0, + .width = 12, + }, + .n2 = { + .reg_off = AO_RTC_ALT_CLK_CNTL0, + .shift = 12, + .width = 12, + }, + .m1 = { + .reg_off = AO_RTC_ALT_CLK_CNTL1, + .shift = 0, + .width = 12, + }, + .m2 = { + .reg_off = AO_RTC_ALT_CLK_CNTL1, + .shift = 12, + .width = 12, + }, + .dual = { + .reg_off = AO_RTC_ALT_CLK_CNTL0, + .shift = 28, + .width = 1, + }, + .table = axg_32k_div_table, + }, + .hw.init = &(struct clk_init_data){ + .name = "axg_ao_32k_div", + .ops = &meson_clk_dualdiv_ops, + .parent_names = (const char *[]){ "axg_ao_32k_pre" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap axg_aoclk_32k_sel = { + .data = &(struct clk_regmap_mux_data) { + .offset = AO_RTC_ALT_CLK_CNTL1, + .mask = 0x1, + .shift = 24, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "axg_ao_32k_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = (const char *[]){ "axg_ao_32k_div", + "axg_ao_32k_pre" }, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap axg_aoclk_32k = { + .data = &(struct clk_regmap_gate_data){ + .offset = AO_RTC_ALT_CLK_CNTL0, + .bit_idx = 30, + }, + .hw.init = &(struct clk_init_data){ + .name = "axg_ao_32k", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "axg_ao_32k_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap axg_aoclk_cts_rtc_oscin = { + .data = &(struct clk_regmap_mux_data) { + .offset = AO_RTI_PWR_CNTL_REG0, + .mask = 0x1, + .shift = 10, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "axg_ao_cts_rtc_oscin", + .ops = &clk_regmap_mux_ops, + .parent_names = (const char *[]){ "axg_ao_32k", + IN_PREFIX "ext_32k-0" }, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + }, +}; + static struct clk_regmap axg_aoclk_clk81 = { .data = &(struct clk_regmap_mux_data) { .offset = AO_RTI_PWR_CNTL_REG0, .mask = 0x1, .shift = 8, + .flags = CLK_MUX_ROUND_CLOSEST, }, .hw.init = &(struct clk_init_data){ .name = "axg_ao_clk81", .ops = &clk_regmap_mux_ro_ops, - .parent_names = (const char *[]){ "clk81", "ao_alt_xtal"}, + .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk", + "axg_ao_cts_rtc_oscin"}, .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -62,7 +203,8 @@ static struct clk_regmap axg_aoclk_saradc_mux = { .hw.init = &(struct clk_init_data){ .name = "axg_ao_saradc_mux", .ops = &clk_regmap_mux_ops, - .parent_names = (const char *[]){ "xtal", "axg_ao_clk81" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal", + "axg_ao_clk81" }, .num_parents = 2, }, }; @@ -106,17 +248,23 @@ static const unsigned int axg_aoclk_reset[] = { }; static struct clk_regmap *axg_aoclk_regmap[] = { - [CLKID_AO_REMOTE] = &axg_aoclk_remote, - [CLKID_AO_I2C_MASTER] = &axg_aoclk_i2c_master, - [CLKID_AO_I2C_SLAVE] = &axg_aoclk_i2c_slave, - [CLKID_AO_UART1] = &axg_aoclk_uart1, - [CLKID_AO_UART2] = &axg_aoclk_uart2, - [CLKID_AO_IR_BLASTER] = &axg_aoclk_ir_blaster, - [CLKID_AO_SAR_ADC] = &axg_aoclk_saradc, - [CLKID_AO_CLK81] = &axg_aoclk_clk81, - [CLKID_AO_SAR_ADC_SEL] = &axg_aoclk_saradc_mux, - [CLKID_AO_SAR_ADC_DIV] = &axg_aoclk_saradc_div, - [CLKID_AO_SAR_ADC_CLK] = &axg_aoclk_saradc_gate, + &axg_aoclk_remote, + &axg_aoclk_i2c_master, + &axg_aoclk_i2c_slave, + &axg_aoclk_uart1, + &axg_aoclk_uart2, + &axg_aoclk_ir_blaster, + &axg_aoclk_saradc, + &axg_aoclk_cts_oscin, + &axg_aoclk_32k_pre, + &axg_aoclk_32k_div, + &axg_aoclk_32k_sel, + &axg_aoclk_32k, + &axg_aoclk_cts_rtc_oscin, + &axg_aoclk_clk81, + &axg_aoclk_saradc_mux, + &axg_aoclk_saradc_div, + &axg_aoclk_saradc_gate, }; static const struct clk_hw_onecell_data axg_aoclk_onecell_data = { @@ -132,10 +280,22 @@ static const struct clk_hw_onecell_data axg_aoclk_onecell_data = { [CLKID_AO_SAR_ADC_SEL] = &axg_aoclk_saradc_mux.hw, [CLKID_AO_SAR_ADC_DIV] = &axg_aoclk_saradc_div.hw, [CLKID_AO_SAR_ADC_CLK] = &axg_aoclk_saradc_gate.hw, + [CLKID_AO_CTS_OSCIN] = &axg_aoclk_cts_oscin.hw, + [CLKID_AO_32K_PRE] = &axg_aoclk_32k_pre.hw, + [CLKID_AO_32K_DIV] = &axg_aoclk_32k_div.hw, + [CLKID_AO_32K_SEL] = &axg_aoclk_32k_sel.hw, + [CLKID_AO_32K] = &axg_aoclk_32k.hw, + [CLKID_AO_CTS_RTC_OSCIN] = &axg_aoclk_cts_rtc_oscin.hw, }, .num = NR_CLKS, }; +static const struct meson_aoclk_input axg_aoclk_inputs[] = { + { .name = "xtal", .required = true }, + { .name = "mpeg-clk", .required = true }, + { .name = "ext-32k-0", .required = false }, +}; + static const struct meson_aoclk_data axg_aoclkc_data = { .reset_reg = AO_RTI_GEN_CNTL_REG0, .num_reset = ARRAY_SIZE(axg_aoclk_reset), @@ -143,6 +303,9 @@ static const struct meson_aoclk_data axg_aoclkc_data = { .num_clks = ARRAY_SIZE(axg_aoclk_regmap), .clks = axg_aoclk_regmap, .hw_data = &axg_aoclk_onecell_data, + .inputs = axg_aoclk_inputs, + .num_inputs = ARRAY_SIZE(axg_aoclk_inputs), + .input_prefix = IN_PREFIX, }; static const struct of_device_id axg_aoclkc_match_table[] = { diff --git a/drivers/clk/meson/axg-aoclk.h b/drivers/clk/meson/axg-aoclk.h index 91384d8dd8444..3cc27e85170f4 100644 --- a/drivers/clk/meson/axg-aoclk.h +++ b/drivers/clk/meson/axg-aoclk.h @@ -10,18 +10,7 @@ #ifndef __AXG_AOCLKC_H #define __AXG_AOCLKC_H -#define NR_CLKS 11 -/* AO Configuration Clock registers offsets - * Register offsets from the data sheet must be multiplied by 4. - */ -#define AO_RTI_PWR_CNTL_REG1 0x0C -#define AO_RTI_PWR_CNTL_REG0 0x10 -#define AO_RTI_GEN_CNTL_REG0 0x40 -#define AO_OSCIN_CNTL 0x58 -#define AO_CRT_CLK_CNTL1 0x68 -#define AO_SAR_CLK 0x90 -#define AO_RTC_ALT_CLK_CNTL0 0x94 -#define AO_RTC_ALT_CLK_CNTL1 0x98 +#define NR_CLKS 17 #include <dt-bindings/clock/axg-aoclkc.h> #include <dt-bindings/reset/axg-aoclkc.h> diff --git a/drivers/clk/meson/axg-audio.c b/drivers/clk/meson/axg-audio.c index 8ac3a22954734..7ab200b6c3bff 100644 --- a/drivers/clk/meson/axg-audio.c +++ b/drivers/clk/meson/axg-audio.c @@ -14,8 +14,11 @@ #include <linux/reset.h> #include <linux/slab.h> -#include "clkc-audio.h" #include "axg-audio.h" +#include "clk-input.h" +#include "clk-regmap.h" +#include "clk-phase.h" +#include "sclk-div.h" #define AXG_MST_IN_COUNT 8 #define AXG_SLV_SCLK_COUNT 10 diff --git a/drivers/clk/meson/axg.c b/drivers/clk/meson/axg.c index 792735d7e46ea..7a8ef80e5f2cb 100644 --- a/drivers/clk/meson/axg.c +++ b/drivers/clk/meson/axg.c @@ -9,16 +9,17 @@ * Author: Qiufang Dai <qiufang.dai@amlogic.com> */ -#include <linux/clk.h> #include <linux/clk-provider.h> #include <linux/init.h> #include <linux/of_device.h> -#include <linux/mfd/syscon.h> #include <linux/platform_device.h> -#include <linux/regmap.h> -#include "clkc.h" +#include "clk-input.h" +#include "clk-regmap.h" +#include "clk-pll.h" +#include "clk-mpll.h" #include "axg.h" +#include "meson-eeclk.h" static DEFINE_SPINLOCK(meson_clk_lock); @@ -58,7 +59,7 @@ static struct clk_regmap axg_fixed_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "fixed_pll_dco", .ops = &meson_clk_pll_ro_ops, - .parent_names = (const char *[]){ "xtal" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, .num_parents = 1, }, }; @@ -113,7 +114,7 @@ static struct clk_regmap axg_sys_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "sys_pll_dco", .ops = &meson_clk_pll_ro_ops, - .parent_names = (const char *[]){ "xtal" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, .num_parents = 1, }, }; @@ -214,7 +215,7 @@ static struct clk_regmap axg_gp0_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "gp0_pll_dco", .ops = &meson_clk_pll_ops, - .parent_names = (const char *[]){ "xtal" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, .num_parents = 1, }, }; @@ -283,7 +284,7 @@ static struct clk_regmap axg_hifi_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "hifi_pll_dco", .ops = &meson_clk_pll_ops, - .parent_names = (const char *[]){ "xtal" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, .num_parents = 1, }, }; @@ -701,7 +702,7 @@ static struct clk_regmap axg_pcie_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "pcie_pll_dco", .ops = &meson_clk_pll_ops, - .parent_names = (const char *[]){ "xtal" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, .num_parents = 1, }, }; @@ -803,7 +804,7 @@ static struct clk_regmap axg_pcie_cml_en1 = { static u32 mux_table_clk81[] = { 0, 2, 3, 4, 5, 6, 7 }; static const char * const clk81_parent_names[] = { - "xtal", "fclk_div7", "mpll1", "mpll2", "fclk_div4", + IN_PREFIX "xtal", "fclk_div7", "mpll1", "mpll2", "fclk_div4", "fclk_div3", "fclk_div5" }; @@ -852,7 +853,7 @@ static struct clk_regmap axg_clk81 = { }; static const char * const axg_sd_emmc_clk0_parent_names[] = { - "xtal", "fclk_div2", "fclk_div3", "fclk_div5", "fclk_div7", + IN_PREFIX "xtal", "fclk_div2", "fclk_div3", "fclk_div5", "fclk_div7", /* * Following these parent clocks, we should also have had mpll2, mpll3 @@ -957,7 +958,7 @@ static struct clk_regmap axg_sd_emmc_c_clk0 = { static u32 mux_table_gen_clk[] = { 0, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, }; static const char * const gen_clk_parent_names[] = { - "xtal", "hifi_pll", "mpll0", "mpll1", "mpll2", "mpll3", + IN_PREFIX "xtal", "hifi_pll", "mpll0", "mpll1", "mpll2", "mpll3", "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7", "gp0_pll", }; @@ -1255,46 +1256,20 @@ static struct clk_regmap *const axg_clk_regmaps[] = { &axg_pcie_pll_od, }; +static const struct meson_eeclkc_data axg_clkc_data = { + .regmap_clks = axg_clk_regmaps, + .regmap_clk_num = ARRAY_SIZE(axg_clk_regmaps), + .hw_onecell_data = &axg_hw_onecell_data, +}; + + static const struct of_device_id clkc_match_table[] = { - { .compatible = "amlogic,axg-clkc" }, + { .compatible = "amlogic,axg-clkc", .data = &axg_clkc_data }, {} }; -static int axg_clkc_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct regmap *map; - int ret, i; - - /* Get the hhi system controller node if available */ - map = syscon_node_to_regmap(of_get_parent(dev->of_node)); - if (IS_ERR(map)) { - dev_err(dev, "failed to get HHI regmap\n"); - return PTR_ERR(map); - } - - /* Populate regmap for the regmap backed clocks */ - for (i = 0; i < ARRAY_SIZE(axg_clk_regmaps); i++) - axg_clk_regmaps[i]->map = map; - - for (i = 0; i < axg_hw_onecell_data.num; i++) { - /* array might be sparse */ - if (!axg_hw_onecell_data.hws[i]) - continue; - - ret = devm_clk_hw_register(dev, axg_hw_onecell_data.hws[i]); - if (ret) { - dev_err(dev, "Clock registration failed\n"); - return ret; - } - } - - return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, - &axg_hw_onecell_data); -} - static struct platform_driver axg_driver = { - .probe = axg_clkc_probe, + .probe = meson_eeclkc_probe, .driver = { .name = "axg-clkc", .of_match_table = clkc_match_table, diff --git a/drivers/clk/meson/clk-dualdiv.c b/drivers/clk/meson/clk-dualdiv.c new file mode 100644 index 0000000000000..c5ca23a5e3e8c --- /dev/null +++ b/drivers/clk/meson/clk-dualdiv.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2017 BayLibre, SAS + * Author: Neil Armstrong <narmstrong@baylibre.com> + * Author: Jerome Brunet <jbrunet@baylibre.com> + */ + +/* + * The AO Domain embeds a dual/divider to generate a more precise + * 32,768KHz clock for low-power suspend mode and CEC. + * ______ ______ + * | | | | + * | Div1 |-| Cnt1 | + * /|______| |______|\ + * -| ______ ______ X--> Out + * \| | | |/ + * | Div2 |-| Cnt2 | + * |______| |______| + * + * The dividing can be switched to single or dual, with a counter + * for each divider to set when the switching is done. + */ + +#include <linux/clk-provider.h> +#include <linux/module.h> + +#include "clk-regmap.h" +#include "clk-dualdiv.h" + +static inline struct meson_clk_dualdiv_data * +meson_clk_dualdiv_data(struct clk_regmap *clk) +{ + return (struct meson_clk_dualdiv_data *)clk->data; +} + +static unsigned long +__dualdiv_param_to_rate(unsigned long parent_rate, + const struct meson_clk_dualdiv_param *p) +{ + if (!p->dual) + return DIV_ROUND_CLOSEST(parent_rate, p->n1); + + return DIV_ROUND_CLOSEST(parent_rate * (p->m1 + p->m2), + p->n1 * p->m1 + p->n2 * p->m2); +} + +static unsigned long meson_clk_dualdiv_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_clk_dualdiv_data *dualdiv = meson_clk_dualdiv_data(clk); + struct meson_clk_dualdiv_param setting; + + setting.dual = meson_parm_read(clk->map, &dualdiv->dual); + setting.n1 = meson_parm_read(clk->map, &dualdiv->n1) + 1; + setting.m1 = meson_parm_read(clk->map, &dualdiv->m1) + 1; + setting.n2 = meson_parm_read(clk->map, &dualdiv->n2) + 1; + setting.m2 = meson_parm_read(clk->map, &dualdiv->m2) + 1; + + return __dualdiv_param_to_rate(parent_rate, &setting); +} + +static const struct meson_clk_dualdiv_param * +__dualdiv_get_setting(unsigned long rate, unsigned long parent_rate, + struct meson_clk_dualdiv_data *dualdiv) +{ + const struct meson_clk_dualdiv_param *table = dualdiv->table; + unsigned long best = 0, now = 0; + unsigned int i, best_i = 0; + + if (!table) + return NULL; + + for (i = 0; table[i].n1; i++) { + now = __dualdiv_param_to_rate(parent_rate, &table[i]); + + /* If we get an exact match, don't bother any further */ + if (now == rate) { + return &table[i]; + } else if (abs(now - rate) < abs(best - rate)) { + best = now; + best_i = i; + } + } + + return (struct meson_clk_dualdiv_param *)&table[best_i]; +} + +static long meson_clk_dualdiv_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_clk_dualdiv_data *dualdiv = meson_clk_dualdiv_data(clk); + const struct meson_clk_dualdiv_param *setting = + __dualdiv_get_setting(rate, *parent_rate, dualdiv); + + if (!setting) + return meson_clk_dualdiv_recalc_rate(hw, *parent_rate); + + return __dualdiv_param_to_rate(*parent_rate, setting); +} + +static int meson_clk_dualdiv_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_clk_dualdiv_data *dualdiv = meson_clk_dualdiv_data(clk); + const struct meson_clk_dualdiv_param *setting = + __dualdiv_get_setting(rate, parent_rate, dualdiv); + + if (!setting) + return -EINVAL; + + meson_parm_write(clk->map, &dualdiv->dual, setting->dual); + meson_parm_write(clk->map, &dualdiv->n1, setting->n1 - 1); + meson_parm_write(clk->map, &dualdiv->m1, setting->m1 - 1); + meson_parm_write(clk->map, &dualdiv->n2, setting->n2 - 1); + meson_parm_write(clk->map, &dualdiv->m2, setting->m2 - 1); + + return 0; +} + +const struct clk_ops meson_clk_dualdiv_ops = { + .recalc_rate = meson_clk_dualdiv_recalc_rate, + .round_rate = meson_clk_dualdiv_round_rate, + .set_rate = meson_clk_dualdiv_set_rate, +}; +EXPORT_SYMBOL_GPL(meson_clk_dualdiv_ops); + +const struct clk_ops meson_clk_dualdiv_ro_ops = { + .recalc_rate = meson_clk_dualdiv_recalc_rate, +}; +EXPORT_SYMBOL_GPL(meson_clk_dualdiv_ro_ops); + +MODULE_DESCRIPTION("Amlogic dual divider driver"); +MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>"); +MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/meson/clk-dualdiv.h b/drivers/clk/meson/clk-dualdiv.h new file mode 100644 index 0000000000000..4aa9390180122 --- /dev/null +++ b/drivers/clk/meson/clk-dualdiv.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019 BayLibre, SAS. + * Author: Jerome Brunet <jbrunet@baylibre.com> + */ + +#ifndef __MESON_CLK_DUALDIV_H +#define __MESON_CLK_DUALDIV_H + +#include <linux/clk-provider.h> +#include "parm.h" + +struct meson_clk_dualdiv_param { + unsigned int n1; + unsigned int n2; + unsigned int m1; + unsigned int m2; + unsigned int dual; +}; + +struct meson_clk_dualdiv_data { + struct parm n1; + struct parm n2; + struct parm m1; + struct parm m2; + struct parm dual; + const struct meson_clk_dualdiv_param *table; +}; + +extern const struct clk_ops meson_clk_dualdiv_ops; +extern const struct clk_ops meson_clk_dualdiv_ro_ops; + +#endif /* __MESON_CLK_DUALDIV_H */ diff --git a/drivers/clk/meson/clk-input.c b/drivers/clk/meson/clk-input.c index 06b3e3bb6a66f..086226e9dba64 100644 --- a/drivers/clk/meson/clk-input.c +++ b/drivers/clk/meson/clk-input.c @@ -7,7 +7,8 @@ #include <linux/clk.h> #include <linux/clk-provider.h> #include <linux/device.h> -#include "clkc.h" +#include <linux/module.h> +#include "clk-input.h" static const struct clk_ops meson_clk_no_ops = {}; @@ -42,3 +43,7 @@ struct clk_hw *meson_clk_hw_register_input(struct device *dev, return ret ? ERR_PTR(ret) : hw; } EXPORT_SYMBOL_GPL(meson_clk_hw_register_input); + +MODULE_DESCRIPTION("Amlogic clock input helper"); +MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/meson/clk-input.h b/drivers/clk/meson/clk-input.h new file mode 100644 index 0000000000000..4a541b9685a66 --- /dev/null +++ b/drivers/clk/meson/clk-input.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019 BayLibre, SAS. + * Author: Jerome Brunet <jbrunet@baylibre.com> + */ + +#ifndef __MESON_CLK_INPUT_H +#define __MESON_CLK_INPUT_H + +#include <linux/clk-provider.h> + +struct device; + +struct clk_hw *meson_clk_hw_register_input(struct device *dev, + const char *of_name, + const char *clk_name, + unsigned long flags); + +#endif /* __MESON_CLK_INPUT_H */ diff --git a/drivers/clk/meson/clk-mpll.c b/drivers/clk/meson/clk-mpll.c index 650f75cc15a93..f76850d99e591 100644 --- a/drivers/clk/meson/clk-mpll.c +++ b/drivers/clk/meson/clk-mpll.c @@ -12,7 +12,11 @@ */ #include <linux/clk-provider.h> -#include "clkc.h" +#include <linux/module.h> +#include <linux/spinlock.h> + +#include "clk-regmap.h" +#include "clk-mpll.h" #define SDM_DEN 16384 #define N2_MIN 4 @@ -138,9 +142,15 @@ const struct clk_ops meson_clk_mpll_ro_ops = { .recalc_rate = mpll_recalc_rate, .round_rate = mpll_round_rate, }; +EXPORT_SYMBOL_GPL(meson_clk_mpll_ro_ops); const struct clk_ops meson_clk_mpll_ops = { .recalc_rate = mpll_recalc_rate, .round_rate = mpll_round_rate, .set_rate = mpll_set_rate, }; +EXPORT_SYMBOL_GPL(meson_clk_mpll_ops); + +MODULE_DESCRIPTION("Amlogic MPLL driver"); +MODULE_AUTHOR("Michael Turquette <mturquette@baylibre.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/meson/clk-mpll.h b/drivers/clk/meson/clk-mpll.h new file mode 100644 index 0000000000000..cf79340006dd7 --- /dev/null +++ b/drivers/clk/meson/clk-mpll.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019 BayLibre, SAS. + * Author: Jerome Brunet <jbrunet@baylibre.com> + */ + +#ifndef __MESON_CLK_MPLL_H +#define __MESON_CLK_MPLL_H + +#include <linux/clk-provider.h> +#include <linux/spinlock.h> + +#include "parm.h" + +struct meson_clk_mpll_data { + struct parm sdm; + struct parm sdm_en; + struct parm n2; + struct parm ssen; + struct parm misc; + spinlock_t *lock; + u8 flags; +}; + +#define CLK_MESON_MPLL_ROUND_CLOSEST BIT(0) + +extern const struct clk_ops meson_clk_mpll_ro_ops; +extern const struct clk_ops meson_clk_mpll_ops; + +#endif /* __MESON_CLK_MPLL_H */ diff --git a/drivers/clk/meson/clk-phase.c b/drivers/clk/meson/clk-phase.c index cba43748ce3df..80c3ada193a43 100644 --- a/drivers/clk/meson/clk-phase.c +++ b/drivers/clk/meson/clk-phase.c @@ -5,7 +5,10 @@ */ #include <linux/clk-provider.h> -#include "clkc.h" +#include <linux/module.h> + +#include "clk-regmap.h" +#include "clk-phase.h" #define phase_step(_width) (360 / (1 << (_width))) @@ -15,13 +18,12 @@ meson_clk_phase_data(struct clk_regmap *clk) return (struct meson_clk_phase_data *)clk->data; } -int meson_clk_degrees_from_val(unsigned int val, unsigned int width) +static int meson_clk_degrees_from_val(unsigned int val, unsigned int width) { return phase_step(width) * val; } -EXPORT_SYMBOL_GPL(meson_clk_degrees_from_val); -unsigned int meson_clk_degrees_to_val(int degrees, unsigned int width) +static unsigned int meson_clk_degrees_to_val(int degrees, unsigned int width) { unsigned int val = DIV_ROUND_CLOSEST(degrees, phase_step(width)); @@ -31,7 +33,6 @@ unsigned int meson_clk_degrees_to_val(int degrees, unsigned int width) */ return val % (1 << width); } -EXPORT_SYMBOL_GPL(meson_clk_degrees_to_val); static int meson_clk_phase_get_phase(struct clk_hw *hw) { @@ -61,3 +62,67 @@ const struct clk_ops meson_clk_phase_ops = { .set_phase = meson_clk_phase_set_phase, }; EXPORT_SYMBOL_GPL(meson_clk_phase_ops); + +/* + * This is a special clock for the audio controller. + * The phase of mst_sclk clock output can be controlled independently + * for the outside world (ph0), the tdmout (ph1) and tdmin (ph2). + * Controlling these 3 phases as just one makes things simpler and + * give the same clock view to all the element on the i2s bus. + * If necessary, we can still control the phase in the tdm block + * which makes these independent control redundant. + */ +static inline struct meson_clk_triphase_data * +meson_clk_triphase_data(struct clk_regmap *clk) +{ + return (struct meson_clk_triphase_data *)clk->data; +} + +static void meson_clk_triphase_sync(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_clk_triphase_data *tph = meson_clk_triphase_data(clk); + unsigned int val; + + /* Get phase 0 and sync it to phase 1 and 2 */ + val = meson_parm_read(clk->map, &tph->ph0); + meson_parm_write(clk->map, &tph->ph1, val); + meson_parm_write(clk->map, &tph->ph2, val); +} + +static int meson_clk_triphase_get_phase(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_clk_triphase_data *tph = meson_clk_triphase_data(clk); + unsigned int val; + + /* Phase are in sync, reading phase 0 is enough */ + val = meson_parm_read(clk->map, &tph->ph0); + + return meson_clk_degrees_from_val(val, tph->ph0.width); +} + +static int meson_clk_triphase_set_phase(struct clk_hw *hw, int degrees) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_clk_triphase_data *tph = meson_clk_triphase_data(clk); + unsigned int val; + + val = meson_clk_degrees_to_val(degrees, tph->ph0.width); + meson_parm_write(clk->map, &tph->ph0, val); + meson_parm_write(clk->map, &tph->ph1, val); + meson_parm_write(clk->map, &tph->ph2, val); + + return 0; +} + +const struct clk_ops meson_clk_triphase_ops = { + .init = meson_clk_triphase_sync, + .get_phase = meson_clk_triphase_get_phase, + .set_phase = meson_clk_triphase_set_phase, +}; +EXPORT_SYMBOL_GPL(meson_clk_triphase_ops); + +MODULE_DESCRIPTION("Amlogic phase driver"); +MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/meson/clk-phase.h b/drivers/clk/meson/clk-phase.h new file mode 100644 index 0000000000000..5579f9ced142a --- /dev/null +++ b/drivers/clk/meson/clk-phase.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019 BayLibre, SAS. + * Author: Jerome Brunet <jbrunet@baylibre.com> + */ + +#ifndef __MESON_CLK_PHASE_H +#define __MESON_CLK_PHASE_H + +#include <linux/clk-provider.h> +#include "parm.h" + +struct meson_clk_phase_data { + struct parm ph; +}; + +struct meson_clk_triphase_data { + struct parm ph0; + struct parm ph1; + struct parm ph2; +}; + +extern const struct clk_ops meson_clk_phase_ops; +extern const struct clk_ops meson_clk_triphase_ops; + +#endif /* __MESON_CLK_PHASE_H */ diff --git a/drivers/clk/meson/clk-pll.c b/drivers/clk/meson/clk-pll.c index afffc1547e20e..41e16dd7272a5 100644 --- a/drivers/clk/meson/clk-pll.c +++ b/drivers/clk/meson/clk-pll.c @@ -32,11 +32,10 @@ #include <linux/io.h> #include <linux/math64.h> #include <linux/module.h> -#include <linux/of_address.h> -#include <linux/slab.h> -#include <linux/string.h> +#include <linux/rational.h> -#include "clkc.h" +#include "clk-regmap.h" +#include "clk-pll.h" static inline struct meson_clk_pll_data * meson_clk_pll_data(struct clk_regmap *clk) @@ -44,12 +43,21 @@ meson_clk_pll_data(struct clk_regmap *clk) return (struct meson_clk_pll_data *)clk->data; } +static int __pll_round_closest_mult(struct meson_clk_pll_data *pll) +{ + if ((pll->flags & CLK_MESON_PLL_ROUND_CLOSEST) && + !MESON_PARM_APPLICABLE(&pll->frac)) + return 1; + + return 0; +} + static unsigned long __pll_params_to_rate(unsigned long parent_rate, - const struct pll_params_table *pllt, - u16 frac, + unsigned int m, unsigned int n, + unsigned int frac, struct meson_clk_pll_data *pll) { - u64 rate = (u64)parent_rate * pllt->m; + u64 rate = (u64)parent_rate * m; if (frac && MESON_PARM_APPLICABLE(&pll->frac)) { u64 frac_rate = (u64)parent_rate * frac; @@ -58,7 +66,7 @@ static unsigned long __pll_params_to_rate(unsigned long parent_rate, (1 << pll->frac.width)); } - return DIV_ROUND_UP_ULL(rate, pllt->n); + return DIV_ROUND_UP_ULL(rate, n); } static unsigned long meson_clk_pll_recalc_rate(struct clk_hw *hw, @@ -66,35 +74,39 @@ static unsigned long meson_clk_pll_recalc_rate(struct clk_hw *hw, { struct clk_regmap *clk = to_clk_regmap(hw); struct meson_clk_pll_data *pll = meson_clk_pll_data(clk); - struct pll_params_table pllt; - u16 frac; + unsigned int m, n, frac; - pllt.n = meson_parm_read(clk->map, &pll->n); - pllt.m = meson_parm_read(clk->map, &pll->m); + n = meson_parm_read(clk->map, &pll->n); + m = meson_parm_read(clk->map, &pll->m); frac = MESON_PARM_APPLICABLE(&pll->frac) ? meson_parm_read(clk->map, &pll->frac) : 0; - return __pll_params_to_rate(parent_rate, &pllt, frac, pll); + return __pll_params_to_rate(parent_rate, m, n, frac, pll); } -static u16 __pll_params_with_frac(unsigned long rate, - unsigned long parent_rate, - const struct pll_params_table *pllt, - struct meson_clk_pll_data *pll) +static unsigned int __pll_params_with_frac(unsigned long rate, + unsigned long parent_rate, + unsigned int m, + unsigned int n, + struct meson_clk_pll_data *pll) { - u16 frac_max = (1 << pll->frac.width); - u64 val = (u64)rate * pllt->n; + unsigned int frac_max = (1 << pll->frac.width); + u64 val = (u64)rate * n; + + /* Bail out if we are already over the requested rate */ + if (rate < parent_rate * m / n) + return 0; if (pll->flags & CLK_MESON_PLL_ROUND_CLOSEST) val = DIV_ROUND_CLOSEST_ULL(val * frac_max, parent_rate); else val = div_u64(val * frac_max, parent_rate); - val -= pllt->m * frac_max; + val -= m * frac_max; - return min((u16)val, (u16)(frac_max - 1)); + return min((unsigned int)val, (frac_max - 1)); } static bool meson_clk_pll_is_better(unsigned long rate, @@ -102,45 +114,123 @@ static bool meson_clk_pll_is_better(unsigned long rate, unsigned long now, struct meson_clk_pll_data *pll) { - if (!(pll->flags & CLK_MESON_PLL_ROUND_CLOSEST) || - MESON_PARM_APPLICABLE(&pll->frac)) { - /* Round down */ - if (now < rate && best < now) - return true; - } else { + if (__pll_round_closest_mult(pll)) { /* Round Closest */ if (abs(now - rate) < abs(best - rate)) return true; + } else { + /* Round down */ + if (now < rate && best < now) + return true; } return false; } -static const struct pll_params_table * -meson_clk_get_pll_settings(unsigned long rate, - unsigned long parent_rate, - struct meson_clk_pll_data *pll) +static int meson_clk_get_pll_table_index(unsigned int index, + unsigned int *m, + unsigned int *n, + struct meson_clk_pll_data *pll) { - const struct pll_params_table *table = pll->table; - unsigned long best = 0, now = 0; - unsigned int i, best_i = 0; + if (!pll->table[index].n) + return -EINVAL; + + *m = pll->table[index].m; + *n = pll->table[index].n; + + return 0; +} + +static unsigned int meson_clk_get_pll_range_m(unsigned long rate, + unsigned long parent_rate, + unsigned int n, + struct meson_clk_pll_data *pll) +{ + u64 val = (u64)rate * n; - if (!table) - return NULL; + if (__pll_round_closest_mult(pll)) + return DIV_ROUND_CLOSEST_ULL(val, parent_rate); - for (i = 0; table[i].n; i++) { - now = __pll_params_to_rate(parent_rate, &table[i], 0, pll); + return div_u64(val, parent_rate); +} - /* If we get an exact match, don't bother any further */ - if (now == rate) { - return &table[i]; - } else if (meson_clk_pll_is_better(rate, best, now, pll)) { +static int meson_clk_get_pll_range_index(unsigned long rate, + unsigned long parent_rate, + unsigned int index, + unsigned int *m, + unsigned int *n, + struct meson_clk_pll_data *pll) +{ + *n = index + 1; + + /* Check the predivider range */ + if (*n >= (1 << pll->n.width)) + return -EINVAL; + + if (*n == 1) { + /* Get the boundaries out the way */ + if (rate <= pll->range->min * parent_rate) { + *m = pll->range->min; + return -ENODATA; + } else if (rate >= pll->range->max * parent_rate) { + *m = pll->range->max; + return -ENODATA; + } + } + + *m = meson_clk_get_pll_range_m(rate, parent_rate, *n, pll); + + /* the pre-divider gives a multiplier too big - stop */ + if (*m >= (1 << pll->m.width)) + return -EINVAL; + + return 0; +} + +static int meson_clk_get_pll_get_index(unsigned long rate, + unsigned long parent_rate, + unsigned int index, + unsigned int *m, + unsigned int *n, + struct meson_clk_pll_data *pll) +{ + if (pll->range) + return meson_clk_get_pll_range_index(rate, parent_rate, + index, m, n, pll); + else if (pll->table) + return meson_clk_get_pll_table_index(index, m, n, pll); + + return -EINVAL; +} + +static int meson_clk_get_pll_settings(unsigned long rate, + unsigned long parent_rate, + unsigned int *best_m, + unsigned int *best_n, + struct meson_clk_pll_data *pll) +{ + unsigned long best = 0, now = 0; + unsigned int i, m, n; + int ret; + + for (i = 0, ret = 0; !ret; i++) { + ret = meson_clk_get_pll_get_index(rate, parent_rate, + i, &m, &n, pll); + if (ret == -EINVAL) + break; + + now = __pll_params_to_rate(parent_rate, m, n, 0, pll); + if (meson_clk_pll_is_better(rate, best, now, pll)) { best = now; - best_i = i; + *best_m = m; + *best_n = n; + + if (now == rate) + break; } } - return (struct pll_params_table *)&table[best_i]; + return best ? 0 : -EINVAL; } static long meson_clk_pll_round_rate(struct clk_hw *hw, unsigned long rate, @@ -148,15 +238,15 @@ static long meson_clk_pll_round_rate(struct clk_hw *hw, unsigned long rate, { struct clk_regmap *clk = to_clk_regmap(hw); struct meson_clk_pll_data *pll = meson_clk_pll_data(clk); - const struct pll_params_table *pllt = - meson_clk_get_pll_settings(rate, *parent_rate, pll); + unsigned int m, n, frac; unsigned long round; - u16 frac; + int ret; - if (!pllt) + ret = meson_clk_get_pll_settings(rate, *parent_rate, &m, &n, pll); + if (ret) return meson_clk_pll_recalc_rate(hw, *parent_rate); - round = __pll_params_to_rate(*parent_rate, pllt, 0, pll); + round = __pll_params_to_rate(*parent_rate, m, n, 0, pll); if (!MESON_PARM_APPLICABLE(&pll->frac) || rate == round) return round; @@ -165,9 +255,9 @@ static long meson_clk_pll_round_rate(struct clk_hw *hw, unsigned long rate, * The rate provided by the setting is not an exact match, let's * try to improve the result using the fractional parameter */ - frac = __pll_params_with_frac(rate, *parent_rate, pllt, pll); + frac = __pll_params_with_frac(rate, *parent_rate, m, n, pll); - return __pll_params_to_rate(*parent_rate, pllt, frac, pll); + return __pll_params_to_rate(*parent_rate, m, n, frac, pll); } static int meson_clk_pll_wait_lock(struct clk_hw *hw) @@ -254,30 +344,27 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate, { struct clk_regmap *clk = to_clk_regmap(hw); struct meson_clk_pll_data *pll = meson_clk_pll_data(clk); - const struct pll_params_table *pllt; - unsigned int enabled; + unsigned int enabled, m, n, frac = 0, ret; unsigned long old_rate; - u16 frac = 0; if (parent_rate == 0 || rate == 0) return -EINVAL; old_rate = rate; - pllt = meson_clk_get_pll_settings(rate, parent_rate, pll); - if (!pllt) - return -EINVAL; + ret = meson_clk_get_pll_settings(rate, parent_rate, &m, &n, pll); + if (ret) + return ret; enabled = meson_parm_read(clk->map, &pll->en); if (enabled) meson_clk_pll_disable(hw); - meson_parm_write(clk->map, &pll->n, pllt->n); - meson_parm_write(clk->map, &pll->m, pllt->m); - + meson_parm_write(clk->map, &pll->n, n); + meson_parm_write(clk->map, &pll->m, m); if (MESON_PARM_APPLICABLE(&pll->frac)) { - frac = __pll_params_with_frac(rate, parent_rate, pllt, pll); + frac = __pll_params_with_frac(rate, parent_rate, m, n, pll); meson_parm_write(clk->map, &pll->frac, frac); } @@ -309,8 +396,15 @@ const struct clk_ops meson_clk_pll_ops = { .enable = meson_clk_pll_enable, .disable = meson_clk_pll_disable }; +EXPORT_SYMBOL_GPL(meson_clk_pll_ops); const struct clk_ops meson_clk_pll_ro_ops = { .recalc_rate = meson_clk_pll_recalc_rate, .is_enabled = meson_clk_pll_is_enabled, }; +EXPORT_SYMBOL_GPL(meson_clk_pll_ro_ops); + +MODULE_DESCRIPTION("Amlogic PLL driver"); +MODULE_AUTHOR("Carlo Caione <carlo@endlessm.com>"); +MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/meson/clk-pll.h b/drivers/clk/meson/clk-pll.h new file mode 100644 index 0000000000000..55af2e285b1b0 --- /dev/null +++ b/drivers/clk/meson/clk-pll.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019 BayLibre, SAS. + * Author: Jerome Brunet <jbrunet@baylibre.com> + */ + +#ifndef __MESON_CLK_PLL_H +#define __MESON_CLK_PLL_H + +#include <linux/clk-provider.h> +#include <linux/regmap.h> +#include "parm.h" + +struct pll_params_table { + unsigned int m; + unsigned int n; +}; + +struct pll_mult_range { + unsigned int min; + unsigned int max; +}; + +#define PLL_PARAMS(_m, _n) \ + { \ + .m = (_m), \ + .n = (_n), \ + } + +#define CLK_MESON_PLL_ROUND_CLOSEST BIT(0) + +struct meson_clk_pll_data { + struct parm en; + struct parm m; + struct parm n; + struct parm frac; + struct parm l; + struct parm rst; + const struct reg_sequence *init_regs; + unsigned int init_count; + const struct pll_params_table *table; + const struct pll_mult_range *range; + u8 flags; +}; + +extern const struct clk_ops meson_clk_pll_ro_ops; +extern const struct clk_ops meson_clk_pll_ops; + +#endif /* __MESON_CLK_PLL_H */ diff --git a/drivers/clk/meson/clk-regmap.c b/drivers/clk/meson/clk-regmap.c index c515f67322a3a..dcd1757cc5dff 100644 --- a/drivers/clk/meson/clk-regmap.c +++ b/drivers/clk/meson/clk-regmap.c @@ -4,6 +4,7 @@ * Author: Jerome Brunet <jbrunet@baylibre.com> */ +#include <linux/module.h> #include "clk-regmap.h" static int clk_regmap_gate_endisable(struct clk_hw *hw, int enable) @@ -180,3 +181,7 @@ const struct clk_ops clk_regmap_mux_ro_ops = { .get_parent = clk_regmap_mux_get_parent, }; EXPORT_SYMBOL_GPL(clk_regmap_mux_ro_ops); + +MODULE_DESCRIPTION("Amlogic regmap backed clock driver"); +MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/meson/clk-regmap.h b/drivers/clk/meson/clk-regmap.h index e9c5728d40eb4..1dd0abe3ba91a 100644 --- a/drivers/clk/meson/clk-regmap.h +++ b/drivers/clk/meson/clk-regmap.h @@ -111,4 +111,24 @@ clk_get_regmap_mux_data(struct clk_regmap *clk) extern const struct clk_ops clk_regmap_mux_ops; extern const struct clk_ops clk_regmap_mux_ro_ops; +#define __MESON_GATE(_name, _reg, _bit, _ops) \ +struct clk_regmap _name = { \ + .data = &(struct clk_regmap_gate_data){ \ + .offset = (_reg), \ + .bit_idx = (_bit), \ + }, \ + .hw.init = &(struct clk_init_data) { \ + .name = #_name, \ + .ops = _ops, \ + .parent_names = (const char *[]){ "clk81" }, \ + .num_parents = 1, \ + .flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED), \ + }, \ +} + +#define MESON_GATE(_name, _reg, _bit) \ + __MESON_GATE(_name, _reg, _bit, &clk_regmap_gate_ops) + +#define MESON_GATE_RO(_name, _reg, _bit) \ + __MESON_GATE(_name, _reg, _bit, &clk_regmap_gate_ro_ops) #endif /* __CLK_REGMAP_H */ diff --git a/drivers/clk/meson/clk-triphase.c b/drivers/clk/meson/clk-triphase.c deleted file mode 100644 index 4a59936251e57..0000000000000 --- a/drivers/clk/meson/clk-triphase.c +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: (GPL-2.0 OR MIT) -/* - * Copyright (c) 2018 BayLibre, SAS. - * Author: Jerome Brunet <jbrunet@baylibre.com> - */ - -#include <linux/clk-provider.h> -#include "clkc-audio.h" - -/* - * This is a special clock for the audio controller. - * The phase of mst_sclk clock output can be controlled independently - * for the outside world (ph0), the tdmout (ph1) and tdmin (ph2). - * Controlling these 3 phases as just one makes things simpler and - * give the same clock view to all the element on the i2s bus. - * If necessary, we can still control the phase in the tdm block - * which makes these independent control redundant. - */ -static inline struct meson_clk_triphase_data * -meson_clk_triphase_data(struct clk_regmap *clk) -{ - return (struct meson_clk_triphase_data *)clk->data; -} - -static void meson_clk_triphase_sync(struct clk_hw *hw) -{ - struct clk_regmap *clk = to_clk_regmap(hw); - struct meson_clk_triphase_data *tph = meson_clk_triphase_data(clk); - unsigned int val; - - /* Get phase 0 and sync it to phase 1 and 2 */ - val = meson_parm_read(clk->map, &tph->ph0); - meson_parm_write(clk->map, &tph->ph1, val); - meson_parm_write(clk->map, &tph->ph2, val); -} - -static int meson_clk_triphase_get_phase(struct clk_hw *hw) -{ - struct clk_regmap *clk = to_clk_regmap(hw); - struct meson_clk_triphase_data *tph = meson_clk_triphase_data(clk); - unsigned int val; - - /* Phase are in sync, reading phase 0 is enough */ - val = meson_parm_read(clk->map, &tph->ph0); - - return meson_clk_degrees_from_val(val, tph->ph0.width); -} - -static int meson_clk_triphase_set_phase(struct clk_hw *hw, int degrees) -{ - struct clk_regmap *clk = to_clk_regmap(hw); - struct meson_clk_triphase_data *tph = meson_clk_triphase_data(clk); - unsigned int val; - - val = meson_clk_degrees_to_val(degrees, tph->ph0.width); - meson_parm_write(clk->map, &tph->ph0, val); - meson_parm_write(clk->map, &tph->ph1, val); - meson_parm_write(clk->map, &tph->ph2, val); - - return 0; -} - -const struct clk_ops meson_clk_triphase_ops = { - .init = meson_clk_triphase_sync, - .get_phase = meson_clk_triphase_get_phase, - .set_phase = meson_clk_triphase_set_phase, -}; -EXPORT_SYMBOL_GPL(meson_clk_triphase_ops); diff --git a/drivers/clk/meson/clkc.h b/drivers/clk/meson/clkc.h deleted file mode 100644 index 6183b22c4bf23..0000000000000 --- a/drivers/clk/meson/clkc.h +++ /dev/null @@ -1,127 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (c) 2015 Endless Mobile, Inc. - * Author: Carlo Caione <carlo@endlessm.com> - */ - -#ifndef __CLKC_H -#define __CLKC_H - -#include <linux/clk-provider.h> -#include "clk-regmap.h" - -#define PMASK(width) GENMASK(width - 1, 0) -#define SETPMASK(width, shift) GENMASK(shift + width - 1, shift) -#define CLRPMASK(width, shift) (~SETPMASK(width, shift)) - -#define PARM_GET(width, shift, reg) \ - (((reg) & SETPMASK(width, shift)) >> (shift)) -#define PARM_SET(width, shift, reg, val) \ - (((reg) & CLRPMASK(width, shift)) | ((val) << (shift))) - -#define MESON_PARM_APPLICABLE(p) (!!((p)->width)) - -struct parm { - u16 reg_off; - u8 shift; - u8 width; -}; - -static inline unsigned int meson_parm_read(struct regmap *map, struct parm *p) -{ - unsigned int val; - - regmap_read(map, p->reg_off, &val); - return PARM_GET(p->width, p->shift, val); -} - -static inline void meson_parm_write(struct regmap *map, struct parm *p, - unsigned int val) -{ - regmap_update_bits(map, p->reg_off, SETPMASK(p->width, p->shift), - val << p->shift); -} - - -struct pll_params_table { - u16 m; - u16 n; -}; - -#define PLL_PARAMS(_m, _n) \ - { \ - .m = (_m), \ - .n = (_n), \ - } - -#define CLK_MESON_PLL_ROUND_CLOSEST BIT(0) - -struct meson_clk_pll_data { - struct parm en; - struct parm m; - struct parm n; - struct parm frac; - struct parm l; - struct parm rst; - const struct reg_sequence *init_regs; - unsigned int init_count; - const struct pll_params_table *table; - u8 flags; -}; - -#define to_meson_clk_pll(_hw) container_of(_hw, struct meson_clk_pll, hw) - -struct meson_clk_mpll_data { - struct parm sdm; - struct parm sdm_en; - struct parm n2; - struct parm ssen; - struct parm misc; - spinlock_t *lock; - u8 flags; -}; - -#define CLK_MESON_MPLL_ROUND_CLOSEST BIT(0) - -struct meson_clk_phase_data { - struct parm ph; -}; - -int meson_clk_degrees_from_val(unsigned int val, unsigned int width); -unsigned int meson_clk_degrees_to_val(int degrees, unsigned int width); - -struct meson_vid_pll_div_data { - struct parm val; - struct parm sel; -}; - -#define MESON_GATE(_name, _reg, _bit) \ -struct clk_regmap _name = { \ - .data = &(struct clk_regmap_gate_data){ \ - .offset = (_reg), \ - .bit_idx = (_bit), \ - }, \ - .hw.init = &(struct clk_init_data) { \ - .name = #_name, \ - .ops = &clk_regmap_gate_ops, \ - .parent_names = (const char *[]){ "clk81" }, \ - .num_parents = 1, \ - .flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED), \ - }, \ -}; - -/* clk_ops */ -extern const struct clk_ops meson_clk_pll_ro_ops; -extern const struct clk_ops meson_clk_pll_ops; -extern const struct clk_ops meson_clk_cpu_ops; -extern const struct clk_ops meson_clk_mpll_ro_ops; -extern const struct clk_ops meson_clk_mpll_ops; -extern const struct clk_ops meson_clk_phase_ops; -extern const struct clk_ops meson_vid_pll_div_ro_ops; - -struct clk_hw *meson_clk_hw_register_input(struct device *dev, - const char *of_name, - const char *clk_name, - unsigned long flags); - -#endif /* __CLKC_H */ diff --git a/drivers/clk/meson/g12a-aoclk.c b/drivers/clk/meson/g12a-aoclk.c new file mode 100644 index 0000000000000..1994e735396b5 --- /dev/null +++ b/drivers/clk/meson/g12a-aoclk.c @@ -0,0 +1,454 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Amlogic Meson-AXG Clock Controller Driver + * + * Copyright (c) 2016 Baylibre SAS. + * Author: Michael Turquette <mturquette@baylibre.com> + * + * Copyright (c) 2019 Baylibre SAS. + * Author: Neil Armstrong <narmstrong@baylibre.com> + */ +#include <linux/clk-provider.h> +#include <linux/platform_device.h> +#include <linux/reset-controller.h> +#include <linux/mfd/syscon.h> +#include "meson-aoclk.h" +#include "g12a-aoclk.h" + +#include "clk-regmap.h" +#include "clk-dualdiv.h" + +#define IN_PREFIX "ao-in-" + +/* + * AO Configuration Clock registers offsets + * Register offsets from the data sheet must be multiplied by 4. + */ +#define AO_RTI_STATUS_REG3 0x0C +#define AO_RTI_PWR_CNTL_REG0 0x10 +#define AO_RTI_GEN_CNTL_REG0 0x40 +#define AO_CLK_GATE0 0x4c +#define AO_CLK_GATE0_SP 0x50 +#define AO_OSCIN_CNTL 0x58 +#define AO_CEC_CLK_CNTL_REG0 0x74 +#define AO_CEC_CLK_CNTL_REG1 0x78 +#define AO_SAR_CLK 0x90 +#define AO_RTC_ALT_CLK_CNTL0 0x94 +#define AO_RTC_ALT_CLK_CNTL1 0x98 + +/* + * Like every other peripheral clock gate in Amlogic Clock drivers, + * we are using CLK_IGNORE_UNUSED here, so we keep the state of the + * bootloader. The goal is to remove this flag at some point. + * Actually removing it will require some extensive test to be done safely. + */ +#define AXG_AO_GATE(_name, _reg, _bit) \ +static struct clk_regmap g12a_aoclk_##_name = { \ + .data = &(struct clk_regmap_gate_data) { \ + .offset = (_reg), \ + .bit_idx = (_bit), \ + }, \ + .hw.init = &(struct clk_init_data) { \ + .name = "g12a_ao_" #_name, \ + .ops = &clk_regmap_gate_ops, \ + .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk" }, \ + .num_parents = 1, \ + .flags = CLK_IGNORE_UNUSED, \ + }, \ +} + +AXG_AO_GATE(ahb, AO_CLK_GATE0, 0); +AXG_AO_GATE(ir_in, AO_CLK_GATE0, 1); +AXG_AO_GATE(i2c_m0, AO_CLK_GATE0, 2); +AXG_AO_GATE(i2c_s0, AO_CLK_GATE0, 3); +AXG_AO_GATE(uart, AO_CLK_GATE0, 4); +AXG_AO_GATE(prod_i2c, AO_CLK_GATE0, 5); +AXG_AO_GATE(uart2, AO_CLK_GATE0, 6); +AXG_AO_GATE(ir_out, AO_CLK_GATE0, 7); +AXG_AO_GATE(saradc, AO_CLK_GATE0, 8); +AXG_AO_GATE(mailbox, AO_CLK_GATE0_SP, 0); +AXG_AO_GATE(m3, AO_CLK_GATE0_SP, 1); +AXG_AO_GATE(ahb_sram, AO_CLK_GATE0_SP, 2); +AXG_AO_GATE(rti, AO_CLK_GATE0_SP, 3); +AXG_AO_GATE(m4_fclk, AO_CLK_GATE0_SP, 4); +AXG_AO_GATE(m4_hclk, AO_CLK_GATE0_SP, 5); + +static struct clk_regmap g12a_aoclk_cts_oscin = { + .data = &(struct clk_regmap_gate_data){ + .offset = AO_RTI_PWR_CNTL_REG0, + .bit_idx = 14, + }, + .hw.init = &(struct clk_init_data){ + .name = "cts_oscin", + .ops = &clk_regmap_gate_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, + .num_parents = 1, + }, +}; + +static const struct meson_clk_dualdiv_param g12a_32k_div_table[] = { + { + .dual = 1, + .n1 = 733, + .m1 = 8, + .n2 = 732, + .m2 = 11, + }, {} +}; + +/* 32k_by_oscin clock */ + +static struct clk_regmap g12a_aoclk_32k_by_oscin_pre = { + .data = &(struct clk_regmap_gate_data){ + .offset = AO_RTC_ALT_CLK_CNTL0, + .bit_idx = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_32k_by_oscin_pre", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "cts_oscin" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_aoclk_32k_by_oscin_div = { + .data = &(struct meson_clk_dualdiv_data){ + .n1 = { + .reg_off = AO_RTC_ALT_CLK_CNTL0, + .shift = 0, + .width = 12, + }, + .n2 = { + .reg_off = AO_RTC_ALT_CLK_CNTL0, + .shift = 12, + .width = 12, + }, + .m1 = { + .reg_off = AO_RTC_ALT_CLK_CNTL1, + .shift = 0, + .width = 12, + }, + .m2 = { + .reg_off = AO_RTC_ALT_CLK_CNTL1, + .shift = 12, + .width = 12, + }, + .dual = { + .reg_off = AO_RTC_ALT_CLK_CNTL0, + .shift = 28, + .width = 1, + }, + .table = g12a_32k_div_table, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_32k_by_oscin_div", + .ops = &meson_clk_dualdiv_ops, + .parent_names = (const char *[]){ "g12a_ao_32k_by_oscin_pre" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_aoclk_32k_by_oscin_sel = { + .data = &(struct clk_regmap_mux_data) { + .offset = AO_RTC_ALT_CLK_CNTL1, + .mask = 0x1, + .shift = 24, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_32k_by_oscin_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = (const char *[]){ "g12a_ao_32k_by_oscin_div", + "g12a_ao_32k_by_oscin_pre" }, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_aoclk_32k_by_oscin = { + .data = &(struct clk_regmap_gate_data){ + .offset = AO_RTC_ALT_CLK_CNTL0, + .bit_idx = 30, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_32k_by_oscin", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "g12a_ao_32k_by_oscin_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +/* cec clock */ + +static struct clk_regmap g12a_aoclk_cec_pre = { + .data = &(struct clk_regmap_gate_data){ + .offset = AO_CEC_CLK_CNTL_REG0, + .bit_idx = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_cec_pre", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "cts_oscin" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_aoclk_cec_div = { + .data = &(struct meson_clk_dualdiv_data){ + .n1 = { + .reg_off = AO_CEC_CLK_CNTL_REG0, + .shift = 0, + .width = 12, + }, + .n2 = { + .reg_off = AO_CEC_CLK_CNTL_REG0, + .shift = 12, + .width = 12, + }, + .m1 = { + .reg_off = AO_CEC_CLK_CNTL_REG1, + .shift = 0, + .width = 12, + }, + .m2 = { + .reg_off = AO_CEC_CLK_CNTL_REG1, + .shift = 12, + .width = 12, + }, + .dual = { + .reg_off = AO_CEC_CLK_CNTL_REG0, + .shift = 28, + .width = 1, + }, + .table = g12a_32k_div_table, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_cec_div", + .ops = &meson_clk_dualdiv_ops, + .parent_names = (const char *[]){ "g12a_ao_cec_pre" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_aoclk_cec_sel = { + .data = &(struct clk_regmap_mux_data) { + .offset = AO_CEC_CLK_CNTL_REG1, + .mask = 0x1, + .shift = 24, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_cec_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = (const char *[]){ "g12a_ao_cec_div", + "g12a_ao_cec_pre" }, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_aoclk_cec = { + .data = &(struct clk_regmap_gate_data){ + .offset = AO_CEC_CLK_CNTL_REG0, + .bit_idx = 30, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_cec", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "g12a_ao_cec_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_aoclk_cts_rtc_oscin = { + .data = &(struct clk_regmap_mux_data) { + .offset = AO_RTI_PWR_CNTL_REG0, + .mask = 0x1, + .shift = 10, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_cts_rtc_oscin", + .ops = &clk_regmap_mux_ops, + .parent_names = (const char *[]){ "g12a_ao_32k_by_oscin", + IN_PREFIX "ext_32k-0" }, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_aoclk_clk81 = { + .data = &(struct clk_regmap_mux_data) { + .offset = AO_RTI_PWR_CNTL_REG0, + .mask = 0x1, + .shift = 8, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_clk81", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk", + "g12a_ao_cts_rtc_oscin"}, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_aoclk_saradc_mux = { + .data = &(struct clk_regmap_mux_data) { + .offset = AO_SAR_CLK, + .mask = 0x3, + .shift = 9, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_saradc_mux", + .ops = &clk_regmap_mux_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal", + "g12a_ao_clk81" }, + .num_parents = 2, + }, +}; + +static struct clk_regmap g12a_aoclk_saradc_div = { + .data = &(struct clk_regmap_div_data) { + .offset = AO_SAR_CLK, + .shift = 0, + .width = 8, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_saradc_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "g12a_ao_saradc_mux" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_aoclk_saradc_gate = { + .data = &(struct clk_regmap_gate_data) { + .offset = AO_SAR_CLK, + .bit_idx = 8, + }, + .hw.init = &(struct clk_init_data){ + .name = "g12a_ao_saradc_gate", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "g12a_ao_saradc_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const unsigned int g12a_aoclk_reset[] = { + [RESET_AO_IR_IN] = 16, + [RESET_AO_UART] = 17, + [RESET_AO_I2C_M] = 18, + [RESET_AO_I2C_S] = 19, + [RESET_AO_SAR_ADC] = 20, + [RESET_AO_UART2] = 22, + [RESET_AO_IR_OUT] = 23, +}; + +static struct clk_regmap *g12a_aoclk_regmap[] = { + &g12a_aoclk_ahb, + &g12a_aoclk_ir_in, + &g12a_aoclk_i2c_m0, + &g12a_aoclk_i2c_s0, + &g12a_aoclk_uart, + &g12a_aoclk_prod_i2c, + &g12a_aoclk_uart2, + &g12a_aoclk_ir_out, + &g12a_aoclk_saradc, + &g12a_aoclk_mailbox, + &g12a_aoclk_m3, + &g12a_aoclk_ahb_sram, + &g12a_aoclk_rti, + &g12a_aoclk_m4_fclk, + &g12a_aoclk_m4_hclk, + &g12a_aoclk_cts_oscin, + &g12a_aoclk_32k_by_oscin_pre, + &g12a_aoclk_32k_by_oscin_div, + &g12a_aoclk_32k_by_oscin_sel, + &g12a_aoclk_32k_by_oscin, + &g12a_aoclk_cec_pre, + &g12a_aoclk_cec_div, + &g12a_aoclk_cec_sel, + &g12a_aoclk_cec, + &g12a_aoclk_cts_rtc_oscin, + &g12a_aoclk_clk81, + &g12a_aoclk_saradc_mux, + &g12a_aoclk_saradc_div, + &g12a_aoclk_saradc_gate, +}; + +static const struct clk_hw_onecell_data g12a_aoclk_onecell_data = { + .hws = { + [CLKID_AO_AHB] = &g12a_aoclk_ahb.hw, + [CLKID_AO_IR_IN] = &g12a_aoclk_ir_in.hw, + [CLKID_AO_I2C_M0] = &g12a_aoclk_i2c_m0.hw, + [CLKID_AO_I2C_S0] = &g12a_aoclk_i2c_s0.hw, + [CLKID_AO_UART] = &g12a_aoclk_uart.hw, + [CLKID_AO_PROD_I2C] = &g12a_aoclk_prod_i2c.hw, + [CLKID_AO_UART2] = &g12a_aoclk_uart2.hw, + [CLKID_AO_IR_OUT] = &g12a_aoclk_ir_out.hw, + [CLKID_AO_SAR_ADC] = &g12a_aoclk_saradc.hw, + [CLKID_AO_MAILBOX] = &g12a_aoclk_mailbox.hw, + [CLKID_AO_M3] = &g12a_aoclk_m3.hw, + [CLKID_AO_AHB_SRAM] = &g12a_aoclk_ahb_sram.hw, + [CLKID_AO_RTI] = &g12a_aoclk_rti.hw, + [CLKID_AO_M4_FCLK] = &g12a_aoclk_m4_fclk.hw, + [CLKID_AO_M4_HCLK] = &g12a_aoclk_m4_hclk.hw, + [CLKID_AO_CLK81] = &g12a_aoclk_clk81.hw, + [CLKID_AO_SAR_ADC_SEL] = &g12a_aoclk_saradc_mux.hw, + [CLKID_AO_SAR_ADC_DIV] = &g12a_aoclk_saradc_div.hw, + [CLKID_AO_SAR_ADC_CLK] = &g12a_aoclk_saradc_gate.hw, + [CLKID_AO_CTS_OSCIN] = &g12a_aoclk_cts_oscin.hw, + [CLKID_AO_32K_PRE] = &g12a_aoclk_32k_by_oscin_pre.hw, + [CLKID_AO_32K_DIV] = &g12a_aoclk_32k_by_oscin_div.hw, + [CLKID_AO_32K_SEL] = &g12a_aoclk_32k_by_oscin_sel.hw, + [CLKID_AO_32K] = &g12a_aoclk_32k_by_oscin.hw, + [CLKID_AO_CEC_PRE] = &g12a_aoclk_cec_pre.hw, + [CLKID_AO_CEC_DIV] = &g12a_aoclk_cec_div.hw, + [CLKID_AO_CEC_SEL] = &g12a_aoclk_cec_sel.hw, + [CLKID_AO_CEC] = &g12a_aoclk_cec.hw, + [CLKID_AO_CTS_RTC_OSCIN] = &g12a_aoclk_cts_rtc_oscin.hw, + }, + .num = NR_CLKS, +}; + +static const struct meson_aoclk_input g12a_aoclk_inputs[] = { + { .name = "xtal", .required = true }, + { .name = "mpeg-clk", .required = true }, + { .name = "ext-32k-0", .required = false }, +}; + +static const struct meson_aoclk_data g12a_aoclkc_data = { + .reset_reg = AO_RTI_GEN_CNTL_REG0, + .num_reset = ARRAY_SIZE(g12a_aoclk_reset), + .reset = g12a_aoclk_reset, + .num_clks = ARRAY_SIZE(g12a_aoclk_regmap), + .clks = g12a_aoclk_regmap, + .hw_data = &g12a_aoclk_onecell_data, + .inputs = g12a_aoclk_inputs, + .num_inputs = ARRAY_SIZE(g12a_aoclk_inputs), + .input_prefix = IN_PREFIX, +}; + +static const struct of_device_id g12a_aoclkc_match_table[] = { + { + .compatible = "amlogic,meson-g12a-aoclkc", + .data = &g12a_aoclkc_data, + }, + { } +}; + +static struct platform_driver g12a_aoclkc_driver = { + .probe = meson_aoclkc_probe, + .driver = { + .name = "g12a-aoclkc", + .of_match_table = g12a_aoclkc_match_table, + }, +}; + +builtin_platform_driver(g12a_aoclkc_driver); diff --git a/drivers/clk/meson/g12a-aoclk.h b/drivers/clk/meson/g12a-aoclk.h new file mode 100644 index 0000000000000..04b0d55066412 --- /dev/null +++ b/drivers/clk/meson/g12a-aoclk.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */ +/* + * Copyright (c) 2019 BayLibre, SAS + * Author: Neil Armstrong <narmstrong@baylibre.com> + */ + +#ifndef __G12A_AOCLKC_H +#define __G12A_AOCLKC_H + +/* + * CLKID index values + * + * These indices are entirely contrived and do not map onto the hardware. + * It has now been decided to expose everything by default in the DT header: + * include/dt-bindings/clock/g12a-aoclkc.h. Only the clocks ids we don't want + * to expose, such as the internal muxes and dividers of composite clocks, + * will remain defined here. + */ +#define CLKID_AO_SAR_ADC_SEL 16 +#define CLKID_AO_SAR_ADC_DIV 17 +#define CLKID_AO_CTS_OSCIN 19 +#define CLKID_AO_32K_PRE 20 +#define CLKID_AO_32K_DIV 21 +#define CLKID_AO_32K_SEL 22 +#define CLKID_AO_CEC_PRE 24 +#define CLKID_AO_CEC_DIV 25 +#define CLKID_AO_CEC_SEL 26 + +#define NR_CLKS 29 + +#include <dt-bindings/clock/g12a-aoclkc.h> +#include <dt-bindings/reset/g12a-aoclkc.h> + +#endif /* __G12A_AOCLKC_H */ diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c new file mode 100644 index 0000000000000..0e1ce8c03259b --- /dev/null +++ b/drivers/clk/meson/g12a.c @@ -0,0 +1,2359 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Amlogic Meson-G12A Clock Controller Driver + * + * Copyright (c) 2016 Baylibre SAS. + * Author: Michael Turquette <mturquette@baylibre.com> + * + * Copyright (c) 2018 Amlogic, inc. + * Author: Qiufang Dai <qiufang.dai@amlogic.com> + * Author: Jian Hu <jian.hu@amlogic.com> + */ + +#include <linux/clk-provider.h> +#include <linux/init.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> + +#include "clk-input.h" +#include "clk-mpll.h" +#include "clk-pll.h" +#include "clk-regmap.h" +#include "vid-pll-div.h" +#include "meson-eeclk.h" +#include "g12a.h" + +static DEFINE_SPINLOCK(meson_clk_lock); + +static struct clk_regmap g12a_fixed_pll_dco = { + .data = &(struct meson_clk_pll_data){ + .en = { + .reg_off = HHI_FIX_PLL_CNTL0, + .shift = 28, + .width = 1, + }, + .m = { + .reg_off = HHI_FIX_PLL_CNTL0, + .shift = 0, + .width = 8, + }, + .n = { + .reg_off = HHI_FIX_PLL_CNTL0, + .shift = 10, + .width = 5, + }, + .frac = { + .reg_off = HHI_FIX_PLL_CNTL1, + .shift = 0, + .width = 17, + }, + .l = { + .reg_off = HHI_FIX_PLL_CNTL0, + .shift = 31, + .width = 1, + }, + .rst = { + .reg_off = HHI_FIX_PLL_CNTL0, + .shift = 29, + .width = 1, + }, + }, + .hw.init = &(struct clk_init_data){ + .name = "fixed_pll_dco", + .ops = &meson_clk_pll_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_fixed_pll = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_FIX_PLL_CNTL0, + .shift = 16, + .width = 2, + .flags = CLK_DIVIDER_POWER_OF_TWO, + }, + .hw.init = &(struct clk_init_data){ + .name = "fixed_pll", + .ops = &clk_regmap_divider_ro_ops, + .parent_names = (const char *[]){ "fixed_pll_dco" }, + .num_parents = 1, + /* + * This clock won't ever change at runtime so + * CLK_SET_RATE_PARENT is not required + */ + }, +}; + +/* + * Internal sys pll emulation configuration parameters + */ +static const struct reg_sequence g12a_sys_init_regs[] = { + { .reg = HHI_SYS_PLL_CNTL1, .def = 0x00000000 }, + { .reg = HHI_SYS_PLL_CNTL2, .def = 0x00000000 }, + { .reg = HHI_SYS_PLL_CNTL3, .def = 0x48681c00 }, + { .reg = HHI_SYS_PLL_CNTL4, .def = 0x88770290 }, + { .reg = HHI_SYS_PLL_CNTL5, .def = 0x39272000 }, + { .reg = HHI_SYS_PLL_CNTL6, .def = 0x56540000 }, +}; + +static struct clk_regmap g12a_sys_pll_dco = { + .data = &(struct meson_clk_pll_data){ + .en = { + .reg_off = HHI_SYS_PLL_CNTL0, + .shift = 28, + .width = 1, + }, + .m = { + .reg_off = HHI_SYS_PLL_CNTL0, + .shift = 0, + .width = 8, + }, + .n = { + .reg_off = HHI_SYS_PLL_CNTL0, + .shift = 10, + .width = 5, + }, + .l = { + .reg_off = HHI_SYS_PLL_CNTL0, + .shift = 31, + .width = 1, + }, + .rst = { + .reg_off = HHI_SYS_PLL_CNTL0, + .shift = 29, + .width = 1, + }, + .init_regs = g12a_sys_init_regs, + .init_count = ARRAY_SIZE(g12a_sys_init_regs), + }, + .hw.init = &(struct clk_init_data){ + .name = "sys_pll_dco", + .ops = &meson_clk_pll_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_sys_pll = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_SYS_PLL_CNTL0, + .shift = 16, + .width = 3, + .flags = CLK_DIVIDER_POWER_OF_TWO, + }, + .hw.init = &(struct clk_init_data){ + .name = "sys_pll", + .ops = &clk_regmap_divider_ro_ops, + .parent_names = (const char *[]){ "sys_pll_dco" }, + .num_parents = 1, + }, +}; + +static const struct pll_mult_range g12a_gp0_pll_mult_range = { + .min = 55, + .max = 255, +}; + +/* + * Internal gp0 pll emulation configuration parameters + */ +static const struct reg_sequence g12a_gp0_init_regs[] = { + { .reg = HHI_GP0_PLL_CNTL1, .def = 0x00000000 }, + { .reg = HHI_GP0_PLL_CNTL2, .def = 0x00000000 }, + { .reg = HHI_GP0_PLL_CNTL3, .def = 0x48681c00 }, + { .reg = HHI_GP0_PLL_CNTL4, .def = 0x33771290 }, + { .reg = HHI_GP0_PLL_CNTL5, .def = 0x39272000 }, + { .reg = HHI_GP0_PLL_CNTL6, .def = 0x56540000 }, +}; + +static struct clk_regmap g12a_gp0_pll_dco = { + .data = &(struct meson_clk_pll_data){ + .en = { + .reg_off = HHI_GP0_PLL_CNTL0, + .shift = 28, + .width = 1, + }, + .m = { + .reg_off = HHI_GP0_PLL_CNTL0, + .shift = 0, + .width = 8, + }, + .n = { + .reg_off = HHI_GP0_PLL_CNTL0, + .shift = 10, + .width = 5, + }, + .frac = { + .reg_off = HHI_GP0_PLL_CNTL1, + .shift = 0, + .width = 17, + }, + .l = { + .reg_off = HHI_GP0_PLL_CNTL0, + .shift = 31, + .width = 1, + }, + .rst = { + .reg_off = HHI_GP0_PLL_CNTL0, + .shift = 29, + .width = 1, + }, + .range = &g12a_gp0_pll_mult_range, + .init_regs = g12a_gp0_init_regs, + .init_count = ARRAY_SIZE(g12a_gp0_init_regs), + }, + .hw.init = &(struct clk_init_data){ + .name = "gp0_pll_dco", + .ops = &meson_clk_pll_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_gp0_pll = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_GP0_PLL_CNTL0, + .shift = 16, + .width = 3, + .flags = (CLK_DIVIDER_POWER_OF_TWO | + CLK_DIVIDER_ROUND_CLOSEST), + }, + .hw.init = &(struct clk_init_data){ + .name = "gp0_pll", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "gp0_pll_dco" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +/* + * Internal hifi pll emulation configuration parameters + */ +static const struct reg_sequence g12a_hifi_init_regs[] = { + { .reg = HHI_HIFI_PLL_CNTL1, .def = 0x00000000 }, + { .reg = HHI_HIFI_PLL_CNTL2, .def = 0x00000000 }, + { .reg = HHI_HIFI_PLL_CNTL3, .def = 0x6a285c00 }, + { .reg = HHI_HIFI_PLL_CNTL4, .def = 0x65771290 }, + { .reg = HHI_HIFI_PLL_CNTL5, .def = 0x39272000 }, + { .reg = HHI_HIFI_PLL_CNTL6, .def = 0x56540000 }, +}; + +static struct clk_regmap g12a_hifi_pll_dco = { + .data = &(struct meson_clk_pll_data){ + .en = { + .reg_off = HHI_HIFI_PLL_CNTL0, + .shift = 28, + .width = 1, + }, + .m = { + .reg_off = HHI_HIFI_PLL_CNTL0, + .shift = 0, + .width = 8, + }, + .n = { + .reg_off = HHI_HIFI_PLL_CNTL0, + .shift = 10, + .width = 5, + }, + .frac = { + .reg_off = HHI_HIFI_PLL_CNTL1, + .shift = 0, + .width = 17, + }, + .l = { + .reg_off = HHI_HIFI_PLL_CNTL0, + .shift = 31, + .width = 1, + }, + .rst = { + .reg_off = HHI_HIFI_PLL_CNTL0, + .shift = 29, + .width = 1, + }, + .range = &g12a_gp0_pll_mult_range, + .init_regs = g12a_hifi_init_regs, + .init_count = ARRAY_SIZE(g12a_hifi_init_regs), + .flags = CLK_MESON_PLL_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "hifi_pll_dco", + .ops = &meson_clk_pll_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_hifi_pll = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_HIFI_PLL_CNTL0, + .shift = 16, + .width = 2, + .flags = (CLK_DIVIDER_POWER_OF_TWO | + CLK_DIVIDER_ROUND_CLOSEST), + }, + .hw.init = &(struct clk_init_data){ + .name = "hifi_pll", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "hifi_pll_dco" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_hdmi_pll_dco = { + .data = &(struct meson_clk_pll_data){ + .en = { + .reg_off = HHI_HDMI_PLL_CNTL0, + .shift = 28, + .width = 1, + }, + .m = { + .reg_off = HHI_HDMI_PLL_CNTL0, + .shift = 0, + .width = 8, + }, + .n = { + .reg_off = HHI_HDMI_PLL_CNTL0, + .shift = 10, + .width = 5, + }, + .frac = { + .reg_off = HHI_HDMI_PLL_CNTL1, + .shift = 0, + .width = 16, + }, + .l = { + .reg_off = HHI_HDMI_PLL_CNTL0, + .shift = 30, + .width = 1, + }, + .rst = { + .reg_off = HHI_HDMI_PLL_CNTL0, + .shift = 29, + .width = 1, + }, + }, + .hw.init = &(struct clk_init_data){ + .name = "hdmi_pll_dco", + .ops = &meson_clk_pll_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, + .num_parents = 1, + /* + * Display directly handle hdmi pll registers ATM, we need + * NOCACHE to keep our view of the clock as accurate as possible + */ + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_regmap g12a_hdmi_pll_od = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_HDMI_PLL_CNTL0, + .shift = 16, + .width = 2, + .flags = CLK_DIVIDER_POWER_OF_TWO, + }, + .hw.init = &(struct clk_init_data){ + .name = "hdmi_pll_od", + .ops = &clk_regmap_divider_ro_ops, + .parent_names = (const char *[]){ "hdmi_pll_dco" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_hdmi_pll_od2 = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_HDMI_PLL_CNTL0, + .shift = 18, + .width = 2, + .flags = CLK_DIVIDER_POWER_OF_TWO, + }, + .hw.init = &(struct clk_init_data){ + .name = "hdmi_pll_od2", + .ops = &clk_regmap_divider_ro_ops, + .parent_names = (const char *[]){ "hdmi_pll_od" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_hdmi_pll = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_HDMI_PLL_CNTL0, + .shift = 20, + .width = 2, + .flags = CLK_DIVIDER_POWER_OF_TWO, + }, + .hw.init = &(struct clk_init_data){ + .name = "hdmi_pll", + .ops = &clk_regmap_divider_ro_ops, + .parent_names = (const char *[]){ "hdmi_pll_od2" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_fixed_factor g12a_fclk_div2_div = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div2_div", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_fclk_div2 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_FIX_PLL_CNTL1, + .bit_idx = 24, + }, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div2", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "fclk_div2_div" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_fclk_div3_div = { + .mult = 1, + .div = 3, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div3_div", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_fclk_div3 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_FIX_PLL_CNTL1, + .bit_idx = 20, + }, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div3", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "fclk_div3_div" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_fclk_div4_div = { + .mult = 1, + .div = 4, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div4_div", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_fclk_div4 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_FIX_PLL_CNTL1, + .bit_idx = 21, + }, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div4", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "fclk_div4_div" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_fclk_div5_div = { + .mult = 1, + .div = 5, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div5_div", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_fclk_div5 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_FIX_PLL_CNTL1, + .bit_idx = 22, + }, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div5", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "fclk_div5_div" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_fclk_div7_div = { + .mult = 1, + .div = 7, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div7_div", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_fclk_div7 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_FIX_PLL_CNTL1, + .bit_idx = 23, + }, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div7", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "fclk_div7_div" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_fclk_div2p5_div = { + .mult = 1, + .div = 5, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div2p5_div", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll_dco" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_fclk_div2p5 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_FIX_PLL_CNTL1, + .bit_idx = 25, + }, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div2p5", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "fclk_div2p5_div" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_mpll_50m_div = { + .mult = 1, + .div = 80, + .hw.init = &(struct clk_init_data){ + .name = "mpll_50m_div", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll_dco" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_mpll_50m = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_FIX_PLL_CNTL3, + .mask = 0x1, + .shift = 5, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpll_50m", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal", + "mpll_50m_div" }, + .num_parents = 2, + }, +}; + +static struct clk_fixed_factor g12a_mpll_prediv = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "mpll_prediv", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll_dco" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_mpll0_div = { + .data = &(struct meson_clk_mpll_data){ + .sdm = { + .reg_off = HHI_MPLL_CNTL1, + .shift = 0, + .width = 14, + }, + .sdm_en = { + .reg_off = HHI_MPLL_CNTL1, + .shift = 30, + .width = 1, + }, + .n2 = { + .reg_off = HHI_MPLL_CNTL1, + .shift = 20, + .width = 9, + }, + .ssen = { + .reg_off = HHI_MPLL_CNTL1, + .shift = 29, + .width = 1, + }, + .lock = &meson_clk_lock, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpll0_div", + .ops = &meson_clk_mpll_ops, + .parent_names = (const char *[]){ "mpll_prediv" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_mpll0 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_MPLL_CNTL1, + .bit_idx = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpll0", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "mpll0_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_mpll1_div = { + .data = &(struct meson_clk_mpll_data){ + .sdm = { + .reg_off = HHI_MPLL_CNTL3, + .shift = 0, + .width = 14, + }, + .sdm_en = { + .reg_off = HHI_MPLL_CNTL3, + .shift = 30, + .width = 1, + }, + .n2 = { + .reg_off = HHI_MPLL_CNTL3, + .shift = 20, + .width = 9, + }, + .ssen = { + .reg_off = HHI_MPLL_CNTL3, + .shift = 29, + .width = 1, + }, + .lock = &meson_clk_lock, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpll1_div", + .ops = &meson_clk_mpll_ops, + .parent_names = (const char *[]){ "mpll_prediv" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_mpll1 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_MPLL_CNTL3, + .bit_idx = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpll1", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "mpll1_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_mpll2_div = { + .data = &(struct meson_clk_mpll_data){ + .sdm = { + .reg_off = HHI_MPLL_CNTL5, + .shift = 0, + .width = 14, + }, + .sdm_en = { + .reg_off = HHI_MPLL_CNTL5, + .shift = 30, + .width = 1, + }, + .n2 = { + .reg_off = HHI_MPLL_CNTL5, + .shift = 20, + .width = 9, + }, + .ssen = { + .reg_off = HHI_MPLL_CNTL5, + .shift = 29, + .width = 1, + }, + .lock = &meson_clk_lock, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpll2_div", + .ops = &meson_clk_mpll_ops, + .parent_names = (const char *[]){ "mpll_prediv" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_mpll2 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_MPLL_CNTL5, + .bit_idx = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpll2", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "mpll2_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_mpll3_div = { + .data = &(struct meson_clk_mpll_data){ + .sdm = { + .reg_off = HHI_MPLL_CNTL7, + .shift = 0, + .width = 14, + }, + .sdm_en = { + .reg_off = HHI_MPLL_CNTL7, + .shift = 30, + .width = 1, + }, + .n2 = { + .reg_off = HHI_MPLL_CNTL7, + .shift = 20, + .width = 9, + }, + .ssen = { + .reg_off = HHI_MPLL_CNTL7, + .shift = 29, + .width = 1, + }, + .lock = &meson_clk_lock, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpll3_div", + .ops = &meson_clk_mpll_ops, + .parent_names = (const char *[]){ "mpll_prediv" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_mpll3 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_MPLL_CNTL7, + .bit_idx = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpll3", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "mpll3_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static u32 mux_table_clk81[] = { 0, 2, 3, 4, 5, 6, 7 }; +static const char * const clk81_parent_names[] = { + IN_PREFIX "xtal", "fclk_div7", "mpll1", "mpll2", "fclk_div4", + "fclk_div3", "fclk_div5" +}; + +static struct clk_regmap g12a_mpeg_clk_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_MPEG_CLK_CNTL, + .mask = 0x7, + .shift = 12, + .table = mux_table_clk81, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpeg_clk_sel", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = clk81_parent_names, + .num_parents = ARRAY_SIZE(clk81_parent_names), + }, +}; + +static struct clk_regmap g12a_mpeg_clk_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_MPEG_CLK_CNTL, + .shift = 0, + .width = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpeg_clk_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "mpeg_clk_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_clk81 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_MPEG_CLK_CNTL, + .bit_idx = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "clk81", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "mpeg_clk_div" }, + .num_parents = 1, + .flags = (CLK_SET_RATE_PARENT | CLK_IS_CRITICAL), + }, +}; + +static const char * const g12a_sd_emmc_clk0_parent_names[] = { + IN_PREFIX "xtal", "fclk_div2", "fclk_div3", "fclk_div5", "fclk_div7", + + /* + * Following these parent clocks, we should also have had mpll2, mpll3 + * and gp0_pll but these clocks are too precious to be used here. All + * the necessary rates for MMC and NAND operation can be acheived using + * g12a_ee_core or fclk_div clocks + */ +}; + +/* SDIO clock */ +static struct clk_regmap g12a_sd_emmc_a_clk0_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SD_EMMC_CLK_CNTL, + .mask = 0x7, + .shift = 9, + }, + .hw.init = &(struct clk_init_data) { + .name = "sd_emmc_a_clk0_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_sd_emmc_clk0_parent_names, + .num_parents = ARRAY_SIZE(g12a_sd_emmc_clk0_parent_names), + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_sd_emmc_a_clk0_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_SD_EMMC_CLK_CNTL, + .shift = 0, + .width = 7, + }, + .hw.init = &(struct clk_init_data) { + .name = "sd_emmc_a_clk0_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "sd_emmc_a_clk0_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_sd_emmc_a_clk0 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_SD_EMMC_CLK_CNTL, + .bit_idx = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "sd_emmc_a_clk0", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "sd_emmc_a_clk0_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +/* SDcard clock */ +static struct clk_regmap g12a_sd_emmc_b_clk0_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SD_EMMC_CLK_CNTL, + .mask = 0x7, + .shift = 25, + }, + .hw.init = &(struct clk_init_data) { + .name = "sd_emmc_b_clk0_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_sd_emmc_clk0_parent_names, + .num_parents = ARRAY_SIZE(g12a_sd_emmc_clk0_parent_names), + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_sd_emmc_b_clk0_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_SD_EMMC_CLK_CNTL, + .shift = 16, + .width = 7, + }, + .hw.init = &(struct clk_init_data) { + .name = "sd_emmc_b_clk0_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "sd_emmc_b_clk0_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_sd_emmc_b_clk0 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_SD_EMMC_CLK_CNTL, + .bit_idx = 23, + }, + .hw.init = &(struct clk_init_data){ + .name = "sd_emmc_b_clk0", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "sd_emmc_b_clk0_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +/* EMMC/NAND clock */ +static struct clk_regmap g12a_sd_emmc_c_clk0_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_NAND_CLK_CNTL, + .mask = 0x7, + .shift = 9, + }, + .hw.init = &(struct clk_init_data) { + .name = "sd_emmc_c_clk0_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_sd_emmc_clk0_parent_names, + .num_parents = ARRAY_SIZE(g12a_sd_emmc_clk0_parent_names), + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_sd_emmc_c_clk0_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_NAND_CLK_CNTL, + .shift = 0, + .width = 7, + }, + .hw.init = &(struct clk_init_data) { + .name = "sd_emmc_c_clk0_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "sd_emmc_c_clk0_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_sd_emmc_c_clk0 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_NAND_CLK_CNTL, + .bit_idx = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "sd_emmc_c_clk0", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "sd_emmc_c_clk0_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +/* VPU Clock */ + +static const char * const g12a_vpu_parent_names[] = { + "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7", + "mpll1", "vid_pll", "hifi_pll", "gp0_pll", +}; + +static struct clk_regmap g12a_vpu_0_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VPU_CLK_CNTL, + .mask = 0x3, + .shift = 9, + }, + .hw.init = &(struct clk_init_data){ + .name = "vpu_0_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_vpu_parent_names, + .num_parents = ARRAY_SIZE(g12a_vpu_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT, + }, +}; + +static struct clk_regmap g12a_vpu_0_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_VPU_CLK_CNTL, + .shift = 0, + .width = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "vpu_0_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "vpu_0_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_vpu_0 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VPU_CLK_CNTL, + .bit_idx = 8, + }, + .hw.init = &(struct clk_init_data) { + .name = "vpu_0", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vpu_0_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vpu_1_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VPU_CLK_CNTL, + .mask = 0x3, + .shift = 25, + }, + .hw.init = &(struct clk_init_data){ + .name = "vpu_1_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_vpu_parent_names, + .num_parents = ARRAY_SIZE(g12a_vpu_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT, + }, +}; + +static struct clk_regmap g12a_vpu_1_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_VPU_CLK_CNTL, + .shift = 16, + .width = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "vpu_1_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "vpu_1_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_vpu_1 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VPU_CLK_CNTL, + .bit_idx = 24, + }, + .hw.init = &(struct clk_init_data) { + .name = "vpu_1", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vpu_1_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vpu = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VPU_CLK_CNTL, + .mask = 1, + .shift = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "vpu", + .ops = &clk_regmap_mux_ops, + /* + * bit 31 selects from 2 possible parents: + * vpu_0 or vpu_1 + */ + .parent_names = (const char *[]){ "vpu_0", "vpu_1" }, + .num_parents = 2, + .flags = CLK_SET_RATE_NO_REPARENT, + }, +}; + +/* VAPB Clock */ + +static const char * const g12a_vapb_parent_names[] = { + "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7", + "mpll1", "vid_pll", "mpll2", "fclk_div2p5", +}; + +static struct clk_regmap g12a_vapb_0_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VAPBCLK_CNTL, + .mask = 0x3, + .shift = 9, + }, + .hw.init = &(struct clk_init_data){ + .name = "vapb_0_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_vapb_parent_names, + .num_parents = ARRAY_SIZE(g12a_vapb_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT, + }, +}; + +static struct clk_regmap g12a_vapb_0_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_VAPBCLK_CNTL, + .shift = 0, + .width = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "vapb_0_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "vapb_0_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_vapb_0 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VAPBCLK_CNTL, + .bit_idx = 8, + }, + .hw.init = &(struct clk_init_data) { + .name = "vapb_0", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vapb_0_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vapb_1_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VAPBCLK_CNTL, + .mask = 0x3, + .shift = 25, + }, + .hw.init = &(struct clk_init_data){ + .name = "vapb_1_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_vapb_parent_names, + .num_parents = ARRAY_SIZE(g12a_vapb_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT, + }, +}; + +static struct clk_regmap g12a_vapb_1_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_VAPBCLK_CNTL, + .shift = 16, + .width = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "vapb_1_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "vapb_1_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_vapb_1 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VAPBCLK_CNTL, + .bit_idx = 24, + }, + .hw.init = &(struct clk_init_data) { + .name = "vapb_1", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vapb_1_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vapb_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VAPBCLK_CNTL, + .mask = 1, + .shift = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "vapb_sel", + .ops = &clk_regmap_mux_ops, + /* + * bit 31 selects from 2 possible parents: + * vapb_0 or vapb_1 + */ + .parent_names = (const char *[]){ "vapb_0", "vapb_1" }, + .num_parents = 2, + .flags = CLK_SET_RATE_NO_REPARENT, + }, +}; + +static struct clk_regmap g12a_vapb = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VAPBCLK_CNTL, + .bit_idx = 30, + }, + .hw.init = &(struct clk_init_data) { + .name = "vapb", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vapb_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +/* Video Clocks */ + +static struct clk_regmap g12a_vid_pll_div = { + .data = &(struct meson_vid_pll_div_data){ + .val = { + .reg_off = HHI_VID_PLL_CLK_DIV, + .shift = 0, + .width = 15, + }, + .sel = { + .reg_off = HHI_VID_PLL_CLK_DIV, + .shift = 16, + .width = 2, + }, + }, + .hw.init = &(struct clk_init_data) { + .name = "vid_pll_div", + .ops = &meson_vid_pll_div_ro_ops, + .parent_names = (const char *[]){ "hdmi_pll" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static const char * const g12a_vid_pll_parent_names[] = { "vid_pll_div", + "hdmi_pll" }; + +static struct clk_regmap g12a_vid_pll_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VID_PLL_CLK_DIV, + .mask = 0x1, + .shift = 18, + }, + .hw.init = &(struct clk_init_data){ + .name = "vid_pll_sel", + .ops = &clk_regmap_mux_ops, + /* + * bit 18 selects from 2 possible parents: + * vid_pll_div or hdmi_pll + */ + .parent_names = g12a_vid_pll_parent_names, + .num_parents = ARRAY_SIZE(g12a_vid_pll_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_regmap g12a_vid_pll = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_PLL_CLK_DIV, + .bit_idx = 19, + }, + .hw.init = &(struct clk_init_data) { + .name = "vid_pll", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vid_pll_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static const char * const g12a_vclk_parent_names[] = { + "vid_pll", "gp0_pll", "hifi_pll", "mpll1", "fclk_div3", "fclk_div4", + "fclk_div5", "fclk_div7" +}; + +static struct clk_regmap g12a_vclk_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VID_CLK_CNTL, + .mask = 0x7, + .shift = 16, + }, + .hw.init = &(struct clk_init_data){ + .name = "vclk_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_vclk_parent_names, + .num_parents = ARRAY_SIZE(g12a_vclk_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_regmap g12a_vclk2_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VIID_CLK_CNTL, + .mask = 0x7, + .shift = 16, + }, + .hw.init = &(struct clk_init_data){ + .name = "vclk2_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_vclk_parent_names, + .num_parents = ARRAY_SIZE(g12a_vclk_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_regmap g12a_vclk_input = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_CLK_DIV, + .bit_idx = 16, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk_input", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk2_input = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VIID_CLK_DIV, + .bit_idx = 16, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk2_input", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk2_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_VID_CLK_DIV, + .shift = 0, + .width = 8, + }, + .hw.init = &(struct clk_init_data){ + .name = "vclk_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "vclk_input" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_regmap g12a_vclk2_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_VIID_CLK_DIV, + .shift = 0, + .width = 8, + }, + .hw.init = &(struct clk_init_data){ + .name = "vclk2_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "vclk2_input" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_regmap g12a_vclk = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_CLK_CNTL, + .bit_idx = 19, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk2 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VIID_CLK_CNTL, + .bit_idx = 19, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk2", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk2_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk_div1 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_CLK_CNTL, + .bit_idx = 0, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk_div1", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk_div2_en = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_CLK_CNTL, + .bit_idx = 1, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk_div2_en", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk_div4_en = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_CLK_CNTL, + .bit_idx = 2, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk_div4_en", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk_div6_en = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_CLK_CNTL, + .bit_idx = 3, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk_div6_en", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk_div12_en = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_CLK_CNTL, + .bit_idx = 4, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk_div12_en", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk2_div1 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VIID_CLK_CNTL, + .bit_idx = 0, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk2_div1", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk2" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk2_div2_en = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VIID_CLK_CNTL, + .bit_idx = 1, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk2_div2_en", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk2" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk2_div4_en = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VIID_CLK_CNTL, + .bit_idx = 2, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk2_div4_en", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk2" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk2_div6_en = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VIID_CLK_CNTL, + .bit_idx = 3, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk2_div6_en", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk2" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_vclk2_div12_en = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VIID_CLK_CNTL, + .bit_idx = 4, + }, + .hw.init = &(struct clk_init_data) { + .name = "vclk2_div12_en", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "vclk2" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_fixed_factor g12a_vclk_div2 = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "vclk_div2", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "vclk_div2_en" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_vclk_div4 = { + .mult = 1, + .div = 4, + .hw.init = &(struct clk_init_data){ + .name = "vclk_div4", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "vclk_div4_en" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_vclk_div6 = { + .mult = 1, + .div = 6, + .hw.init = &(struct clk_init_data){ + .name = "vclk_div6", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "vclk_div6_en" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_vclk_div12 = { + .mult = 1, + .div = 12, + .hw.init = &(struct clk_init_data){ + .name = "vclk_div12", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "vclk_div12_en" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_vclk2_div2 = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "vclk2_div2", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "vclk2_div2_en" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_vclk2_div4 = { + .mult = 1, + .div = 4, + .hw.init = &(struct clk_init_data){ + .name = "vclk2_div4", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "vclk2_div4_en" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_vclk2_div6 = { + .mult = 1, + .div = 6, + .hw.init = &(struct clk_init_data){ + .name = "vclk2_div6", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "vclk2_div6_en" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12a_vclk2_div12 = { + .mult = 1, + .div = 12, + .hw.init = &(struct clk_init_data){ + .name = "vclk2_div12", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "vclk2_div12_en" }, + .num_parents = 1, + }, +}; + +static u32 mux_table_cts_sel[] = { 0, 1, 2, 3, 4, 8, 9, 10, 11, 12 }; +static const char * const g12a_cts_parent_names[] = { + "vclk_div1", "vclk_div2", "vclk_div4", "vclk_div6", + "vclk_div12", "vclk2_div1", "vclk2_div2", "vclk2_div4", + "vclk2_div6", "vclk2_div12" +}; + +static struct clk_regmap g12a_cts_enci_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VID_CLK_DIV, + .mask = 0xf, + .shift = 28, + .table = mux_table_cts_sel, + }, + .hw.init = &(struct clk_init_data){ + .name = "cts_enci_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_cts_parent_names, + .num_parents = ARRAY_SIZE(g12a_cts_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_regmap g12a_cts_encp_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VID_CLK_DIV, + .mask = 0xf, + .shift = 20, + .table = mux_table_cts_sel, + }, + .hw.init = &(struct clk_init_data){ + .name = "cts_encp_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_cts_parent_names, + .num_parents = ARRAY_SIZE(g12a_cts_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_regmap g12a_cts_vdac_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_VIID_CLK_DIV, + .mask = 0xf, + .shift = 28, + .table = mux_table_cts_sel, + }, + .hw.init = &(struct clk_init_data){ + .name = "cts_vdac_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_cts_parent_names, + .num_parents = ARRAY_SIZE(g12a_cts_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +/* TOFIX: add support for cts_tcon */ +static u32 mux_table_hdmi_tx_sel[] = { 0, 1, 2, 3, 4, 8, 9, 10, 11, 12 }; +static const char * const g12a_cts_hdmi_tx_parent_names[] = { + "vclk_div1", "vclk_div2", "vclk_div4", "vclk_div6", + "vclk_div12", "vclk2_div1", "vclk2_div2", "vclk2_div4", + "vclk2_div6", "vclk2_div12" +}; + +static struct clk_regmap g12a_hdmi_tx_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_HDMI_CLK_CNTL, + .mask = 0xf, + .shift = 16, + .table = mux_table_hdmi_tx_sel, + }, + .hw.init = &(struct clk_init_data){ + .name = "hdmi_tx_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_cts_hdmi_tx_parent_names, + .num_parents = ARRAY_SIZE(g12a_cts_hdmi_tx_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_regmap g12a_cts_enci = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_CLK_CNTL2, + .bit_idx = 0, + }, + .hw.init = &(struct clk_init_data) { + .name = "cts_enci", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "cts_enci_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_cts_encp = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_CLK_CNTL2, + .bit_idx = 2, + }, + .hw.init = &(struct clk_init_data) { + .name = "cts_encp", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "cts_encp_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_cts_vdac = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_CLK_CNTL2, + .bit_idx = 4, + }, + .hw.init = &(struct clk_init_data) { + .name = "cts_vdac", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "cts_vdac_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +static struct clk_regmap g12a_hdmi_tx = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_VID_CLK_CNTL2, + .bit_idx = 5, + }, + .hw.init = &(struct clk_init_data) { + .name = "hdmi_tx", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "hdmi_tx_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +/* HDMI Clocks */ + +static const char * const g12a_hdmi_parent_names[] = { + IN_PREFIX "xtal", "fclk_div4", "fclk_div3", "fclk_div5" +}; + +static struct clk_regmap g12a_hdmi_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_HDMI_CLK_CNTL, + .mask = 0x3, + .shift = 9, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "hdmi_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_hdmi_parent_names, + .num_parents = ARRAY_SIZE(g12a_hdmi_parent_names), + .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_regmap g12a_hdmi_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_HDMI_CLK_CNTL, + .shift = 0, + .width = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "hdmi_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "hdmi_sel" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_regmap g12a_hdmi = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_HDMI_CLK_CNTL, + .bit_idx = 8, + }, + .hw.init = &(struct clk_init_data) { + .name = "hdmi", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "hdmi_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, +}; + +/* + * The MALI IP is clocked by two identical clocks (mali_0 and mali_1) + * muxed by a glitch-free switch. + */ + +static const char * const g12a_mali_0_1_parent_names[] = { + IN_PREFIX "xtal", "gp0_pll", "hihi_pll", "fclk_div2p5", + "fclk_div3", "fclk_div4", "fclk_div5", "fclk_div7" +}; + +static struct clk_regmap g12a_mali_0_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_MALI_CLK_CNTL, + .mask = 0x7, + .shift = 9, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_0_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_mali_0_1_parent_names, + .num_parents = 8, + .flags = CLK_SET_RATE_NO_REPARENT, + }, +}; + +static struct clk_regmap g12a_mali_0_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_MALI_CLK_CNTL, + .shift = 0, + .width = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_0_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "mali_0_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_NO_REPARENT, + }, +}; + +static struct clk_regmap g12a_mali_0 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_MALI_CLK_CNTL, + .bit_idx = 8, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_0", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "mali_0_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap g12a_mali_1_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_MALI_CLK_CNTL, + .mask = 0x7, + .shift = 25, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_1_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_mali_0_1_parent_names, + .num_parents = 8, + .flags = CLK_SET_RATE_NO_REPARENT, + }, +}; + +static struct clk_regmap g12a_mali_1_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_MALI_CLK_CNTL, + .shift = 16, + .width = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_1_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "mali_1_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_NO_REPARENT, + }, +}; + +static struct clk_regmap g12a_mali_1 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_MALI_CLK_CNTL, + .bit_idx = 24, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_1", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "mali_1_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const char * const g12a_mali_parent_names[] = { + "mali_0", "mali_1" +}; + +static struct clk_regmap g12a_mali = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_MALI_CLK_CNTL, + .mask = 1, + .shift = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali", + .ops = &clk_regmap_mux_ops, + .parent_names = g12a_mali_parent_names, + .num_parents = 2, + .flags = CLK_SET_RATE_NO_REPARENT, + }, +}; + +/* Everything Else (EE) domain gates */ +static MESON_GATE(g12a_ddr, HHI_GCLK_MPEG0, 0); +static MESON_GATE(g12a_dos, HHI_GCLK_MPEG0, 1); +static MESON_GATE(g12a_audio_locker, HHI_GCLK_MPEG0, 2); +static MESON_GATE(g12a_mipi_dsi_host, HHI_GCLK_MPEG0, 3); +static MESON_GATE(g12a_eth_phy, HHI_GCLK_MPEG0, 4); +static MESON_GATE(g12a_isa, HHI_GCLK_MPEG0, 5); +static MESON_GATE(g12a_pl301, HHI_GCLK_MPEG0, 6); +static MESON_GATE(g12a_periphs, HHI_GCLK_MPEG0, 7); +static MESON_GATE(g12a_spicc_0, HHI_GCLK_MPEG0, 8); +static MESON_GATE(g12a_i2c, HHI_GCLK_MPEG0, 9); +static MESON_GATE(g12a_sana, HHI_GCLK_MPEG0, 10); +static MESON_GATE(g12a_sd, HHI_GCLK_MPEG0, 11); +static MESON_GATE(g12a_rng0, HHI_GCLK_MPEG0, 12); +static MESON_GATE(g12a_uart0, HHI_GCLK_MPEG0, 13); +static MESON_GATE(g12a_spicc_1, HHI_GCLK_MPEG0, 14); +static MESON_GATE(g12a_hiu_reg, HHI_GCLK_MPEG0, 19); +static MESON_GATE(g12a_mipi_dsi_phy, HHI_GCLK_MPEG0, 20); +static MESON_GATE(g12a_assist_misc, HHI_GCLK_MPEG0, 23); +static MESON_GATE(g12a_emmc_a, HHI_GCLK_MPEG0, 4); +static MESON_GATE(g12a_emmc_b, HHI_GCLK_MPEG0, 25); +static MESON_GATE(g12a_emmc_c, HHI_GCLK_MPEG0, 26); +static MESON_GATE(g12a_audio_codec, HHI_GCLK_MPEG0, 28); + +static MESON_GATE(g12a_audio, HHI_GCLK_MPEG1, 0); +static MESON_GATE(g12a_eth_core, HHI_GCLK_MPEG1, 3); +static MESON_GATE(g12a_demux, HHI_GCLK_MPEG1, 4); +static MESON_GATE(g12a_audio_ififo, HHI_GCLK_MPEG1, 11); +static MESON_GATE(g12a_adc, HHI_GCLK_MPEG1, 13); +static MESON_GATE(g12a_uart1, HHI_GCLK_MPEG1, 16); +static MESON_GATE(g12a_g2d, HHI_GCLK_MPEG1, 20); +static MESON_GATE(g12a_reset, HHI_GCLK_MPEG1, 23); +static MESON_GATE(g12a_pcie_comb, HHI_GCLK_MPEG1, 24); +static MESON_GATE(g12a_parser, HHI_GCLK_MPEG1, 25); +static MESON_GATE(g12a_usb_general, HHI_GCLK_MPEG1, 26); +static MESON_GATE(g12a_pcie_phy, HHI_GCLK_MPEG1, 27); +static MESON_GATE(g12a_ahb_arb0, HHI_GCLK_MPEG1, 29); + +static MESON_GATE(g12a_ahb_data_bus, HHI_GCLK_MPEG2, 1); +static MESON_GATE(g12a_ahb_ctrl_bus, HHI_GCLK_MPEG2, 2); +static MESON_GATE(g12a_htx_hdcp22, HHI_GCLK_MPEG2, 3); +static MESON_GATE(g12a_htx_pclk, HHI_GCLK_MPEG2, 4); +static MESON_GATE(g12a_bt656, HHI_GCLK_MPEG2, 6); +static MESON_GATE(g12a_usb1_to_ddr, HHI_GCLK_MPEG2, 8); +static MESON_GATE(g12a_mmc_pclk, HHI_GCLK_MPEG2, 11); +static MESON_GATE(g12a_uart2, HHI_GCLK_MPEG2, 15); +static MESON_GATE(g12a_vpu_intr, HHI_GCLK_MPEG2, 25); +static MESON_GATE(g12a_gic, HHI_GCLK_MPEG2, 30); + +static MESON_GATE(g12a_vclk2_venci0, HHI_GCLK_OTHER, 1); +static MESON_GATE(g12a_vclk2_venci1, HHI_GCLK_OTHER, 2); +static MESON_GATE(g12a_vclk2_vencp0, HHI_GCLK_OTHER, 3); +static MESON_GATE(g12a_vclk2_vencp1, HHI_GCLK_OTHER, 4); +static MESON_GATE(g12a_vclk2_venct0, HHI_GCLK_OTHER, 5); +static MESON_GATE(g12a_vclk2_venct1, HHI_GCLK_OTHER, 6); +static MESON_GATE(g12a_vclk2_other, HHI_GCLK_OTHER, 7); +static MESON_GATE(g12a_vclk2_enci, HHI_GCLK_OTHER, 8); +static MESON_GATE(g12a_vclk2_encp, HHI_GCLK_OTHER, 9); +static MESON_GATE(g12a_dac_clk, HHI_GCLK_OTHER, 10); +static MESON_GATE(g12a_aoclk_gate, HHI_GCLK_OTHER, 14); +static MESON_GATE(g12a_iec958_gate, HHI_GCLK_OTHER, 16); +static MESON_GATE(g12a_enc480p, HHI_GCLK_OTHER, 20); +static MESON_GATE(g12a_rng1, HHI_GCLK_OTHER, 21); +static MESON_GATE(g12a_vclk2_enct, HHI_GCLK_OTHER, 22); +static MESON_GATE(g12a_vclk2_encl, HHI_GCLK_OTHER, 23); +static MESON_GATE(g12a_vclk2_venclmmc, HHI_GCLK_OTHER, 24); +static MESON_GATE(g12a_vclk2_vencl, HHI_GCLK_OTHER, 25); +static MESON_GATE(g12a_vclk2_other1, HHI_GCLK_OTHER, 26); + +static MESON_GATE_RO(g12a_dma, HHI_GCLK_OTHER2, 0); +static MESON_GATE_RO(g12a_efuse, HHI_GCLK_OTHER2, 1); +static MESON_GATE_RO(g12a_rom_boot, HHI_GCLK_OTHER2, 2); +static MESON_GATE_RO(g12a_reset_sec, HHI_GCLK_OTHER2, 3); +static MESON_GATE_RO(g12a_sec_ahb_apb3, HHI_GCLK_OTHER2, 4); + +/* Array of all clocks provided by this provider */ +static struct clk_hw_onecell_data g12a_hw_onecell_data = { + .hws = { + [CLKID_SYS_PLL] = &g12a_sys_pll.hw, + [CLKID_FIXED_PLL] = &g12a_fixed_pll.hw, + [CLKID_FCLK_DIV2] = &g12a_fclk_div2.hw, + [CLKID_FCLK_DIV3] = &g12a_fclk_div3.hw, + [CLKID_FCLK_DIV4] = &g12a_fclk_div4.hw, + [CLKID_FCLK_DIV5] = &g12a_fclk_div5.hw, + [CLKID_FCLK_DIV7] = &g12a_fclk_div7.hw, + [CLKID_FCLK_DIV2P5] = &g12a_fclk_div2p5.hw, + [CLKID_GP0_PLL] = &g12a_gp0_pll.hw, + [CLKID_MPEG_SEL] = &g12a_mpeg_clk_sel.hw, + [CLKID_MPEG_DIV] = &g12a_mpeg_clk_div.hw, + [CLKID_CLK81] = &g12a_clk81.hw, + [CLKID_MPLL0] = &g12a_mpll0.hw, + [CLKID_MPLL1] = &g12a_mpll1.hw, + [CLKID_MPLL2] = &g12a_mpll2.hw, + [CLKID_MPLL3] = &g12a_mpll3.hw, + [CLKID_DDR] = &g12a_ddr.hw, + [CLKID_DOS] = &g12a_dos.hw, + [CLKID_AUDIO_LOCKER] = &g12a_audio_locker.hw, + [CLKID_MIPI_DSI_HOST] = &g12a_mipi_dsi_host.hw, + [CLKID_ETH_PHY] = &g12a_eth_phy.hw, + [CLKID_ISA] = &g12a_isa.hw, + [CLKID_PL301] = &g12a_pl301.hw, + [CLKID_PERIPHS] = &g12a_periphs.hw, + [CLKID_SPICC0] = &g12a_spicc_0.hw, + [CLKID_I2C] = &g12a_i2c.hw, + [CLKID_SANA] = &g12a_sana.hw, + [CLKID_SD] = &g12a_sd.hw, + [CLKID_RNG0] = &g12a_rng0.hw, + [CLKID_UART0] = &g12a_uart0.hw, + [CLKID_SPICC1] = &g12a_spicc_1.hw, + [CLKID_HIU_IFACE] = &g12a_hiu_reg.hw, + [CLKID_MIPI_DSI_PHY] = &g12a_mipi_dsi_phy.hw, + [CLKID_ASSIST_MISC] = &g12a_assist_misc.hw, + [CLKID_SD_EMMC_A] = &g12a_emmc_a.hw, + [CLKID_SD_EMMC_B] = &g12a_emmc_b.hw, + [CLKID_SD_EMMC_C] = &g12a_emmc_c.hw, + [CLKID_AUDIO_CODEC] = &g12a_audio_codec.hw, + [CLKID_AUDIO] = &g12a_audio.hw, + [CLKID_ETH] = &g12a_eth_core.hw, + [CLKID_DEMUX] = &g12a_demux.hw, + [CLKID_AUDIO_IFIFO] = &g12a_audio_ififo.hw, + [CLKID_ADC] = &g12a_adc.hw, + [CLKID_UART1] = &g12a_uart1.hw, + [CLKID_G2D] = &g12a_g2d.hw, + [CLKID_RESET] = &g12a_reset.hw, + [CLKID_PCIE_COMB] = &g12a_pcie_comb.hw, + [CLKID_PARSER] = &g12a_parser.hw, + [CLKID_USB] = &g12a_usb_general.hw, + [CLKID_PCIE_PHY] = &g12a_pcie_phy.hw, + [CLKID_AHB_ARB0] = &g12a_ahb_arb0.hw, + [CLKID_AHB_DATA_BUS] = &g12a_ahb_data_bus.hw, + [CLKID_AHB_CTRL_BUS] = &g12a_ahb_ctrl_bus.hw, + [CLKID_HTX_HDCP22] = &g12a_htx_hdcp22.hw, + [CLKID_HTX_PCLK] = &g12a_htx_pclk.hw, + [CLKID_BT656] = &g12a_bt656.hw, + [CLKID_USB1_DDR_BRIDGE] = &g12a_usb1_to_ddr.hw, + [CLKID_MMC_PCLK] = &g12a_mmc_pclk.hw, + [CLKID_UART2] = &g12a_uart2.hw, + [CLKID_VPU_INTR] = &g12a_vpu_intr.hw, + [CLKID_GIC] = &g12a_gic.hw, + [CLKID_SD_EMMC_A_CLK0_SEL] = &g12a_sd_emmc_a_clk0_sel.hw, + [CLKID_SD_EMMC_A_CLK0_DIV] = &g12a_sd_emmc_a_clk0_div.hw, + [CLKID_SD_EMMC_A_CLK0] = &g12a_sd_emmc_a_clk0.hw, + [CLKID_SD_EMMC_B_CLK0_SEL] = &g12a_sd_emmc_b_clk0_sel.hw, + [CLKID_SD_EMMC_B_CLK0_DIV] = &g12a_sd_emmc_b_clk0_div.hw, + [CLKID_SD_EMMC_B_CLK0] = &g12a_sd_emmc_b_clk0.hw, + [CLKID_SD_EMMC_C_CLK0_SEL] = &g12a_sd_emmc_c_clk0_sel.hw, + [CLKID_SD_EMMC_C_CLK0_DIV] = &g12a_sd_emmc_c_clk0_div.hw, + [CLKID_SD_EMMC_C_CLK0] = &g12a_sd_emmc_c_clk0.hw, + [CLKID_MPLL0_DIV] = &g12a_mpll0_div.hw, + [CLKID_MPLL1_DIV] = &g12a_mpll1_div.hw, + [CLKID_MPLL2_DIV] = &g12a_mpll2_div.hw, + [CLKID_MPLL3_DIV] = &g12a_mpll3_div.hw, + [CLKID_FCLK_DIV2_DIV] = &g12a_fclk_div2_div.hw, + [CLKID_FCLK_DIV3_DIV] = &g12a_fclk_div3_div.hw, + [CLKID_FCLK_DIV4_DIV] = &g12a_fclk_div4_div.hw, + [CLKID_FCLK_DIV5_DIV] = &g12a_fclk_div5_div.hw, + [CLKID_FCLK_DIV7_DIV] = &g12a_fclk_div7_div.hw, + [CLKID_FCLK_DIV2P5_DIV] = &g12a_fclk_div2p5_div.hw, + [CLKID_HIFI_PLL] = &g12a_hifi_pll.hw, + [CLKID_VCLK2_VENCI0] = &g12a_vclk2_venci0.hw, + [CLKID_VCLK2_VENCI1] = &g12a_vclk2_venci1.hw, + [CLKID_VCLK2_VENCP0] = &g12a_vclk2_vencp0.hw, + [CLKID_VCLK2_VENCP1] = &g12a_vclk2_vencp1.hw, + [CLKID_VCLK2_VENCT0] = &g12a_vclk2_venct0.hw, + [CLKID_VCLK2_VENCT1] = &g12a_vclk2_venct1.hw, + [CLKID_VCLK2_OTHER] = &g12a_vclk2_other.hw, + [CLKID_VCLK2_ENCI] = &g12a_vclk2_enci.hw, + [CLKID_VCLK2_ENCP] = &g12a_vclk2_encp.hw, + [CLKID_DAC_CLK] = &g12a_dac_clk.hw, + [CLKID_AOCLK] = &g12a_aoclk_gate.hw, + [CLKID_IEC958] = &g12a_iec958_gate.hw, + [CLKID_ENC480P] = &g12a_enc480p.hw, + [CLKID_RNG1] = &g12a_rng1.hw, + [CLKID_VCLK2_ENCT] = &g12a_vclk2_enct.hw, + [CLKID_VCLK2_ENCL] = &g12a_vclk2_encl.hw, + [CLKID_VCLK2_VENCLMMC] = &g12a_vclk2_venclmmc.hw, + [CLKID_VCLK2_VENCL] = &g12a_vclk2_vencl.hw, + [CLKID_VCLK2_OTHER1] = &g12a_vclk2_other1.hw, + [CLKID_FIXED_PLL_DCO] = &g12a_fixed_pll_dco.hw, + [CLKID_SYS_PLL_DCO] = &g12a_sys_pll_dco.hw, + [CLKID_GP0_PLL_DCO] = &g12a_gp0_pll_dco.hw, + [CLKID_HIFI_PLL_DCO] = &g12a_hifi_pll_dco.hw, + [CLKID_DMA] = &g12a_dma.hw, + [CLKID_EFUSE] = &g12a_efuse.hw, + [CLKID_ROM_BOOT] = &g12a_rom_boot.hw, + [CLKID_RESET_SEC] = &g12a_reset_sec.hw, + [CLKID_SEC_AHB_APB3] = &g12a_sec_ahb_apb3.hw, + [CLKID_MPLL_PREDIV] = &g12a_mpll_prediv.hw, + [CLKID_VPU_0_SEL] = &g12a_vpu_0_sel.hw, + [CLKID_VPU_0_DIV] = &g12a_vpu_0_div.hw, + [CLKID_VPU_0] = &g12a_vpu_0.hw, + [CLKID_VPU_1_SEL] = &g12a_vpu_1_sel.hw, + [CLKID_VPU_1_DIV] = &g12a_vpu_1_div.hw, + [CLKID_VPU_1] = &g12a_vpu_1.hw, + [CLKID_VPU] = &g12a_vpu.hw, + [CLKID_VAPB_0_SEL] = &g12a_vapb_0_sel.hw, + [CLKID_VAPB_0_DIV] = &g12a_vapb_0_div.hw, + [CLKID_VAPB_0] = &g12a_vapb_0.hw, + [CLKID_VAPB_1_SEL] = &g12a_vapb_1_sel.hw, + [CLKID_VAPB_1_DIV] = &g12a_vapb_1_div.hw, + [CLKID_VAPB_1] = &g12a_vapb_1.hw, + [CLKID_VAPB_SEL] = &g12a_vapb_sel.hw, + [CLKID_VAPB] = &g12a_vapb.hw, + [CLKID_HDMI_PLL_DCO] = &g12a_hdmi_pll_dco.hw, + [CLKID_HDMI_PLL_OD] = &g12a_hdmi_pll_od.hw, + [CLKID_HDMI_PLL_OD2] = &g12a_hdmi_pll_od2.hw, + [CLKID_HDMI_PLL] = &g12a_hdmi_pll.hw, + [CLKID_VID_PLL] = &g12a_vid_pll_div.hw, + [CLKID_VID_PLL_SEL] = &g12a_vid_pll_sel.hw, + [CLKID_VID_PLL_DIV] = &g12a_vid_pll.hw, + [CLKID_VCLK_SEL] = &g12a_vclk_sel.hw, + [CLKID_VCLK2_SEL] = &g12a_vclk2_sel.hw, + [CLKID_VCLK_INPUT] = &g12a_vclk_input.hw, + [CLKID_VCLK2_INPUT] = &g12a_vclk2_input.hw, + [CLKID_VCLK_DIV] = &g12a_vclk_div.hw, + [CLKID_VCLK2_DIV] = &g12a_vclk2_div.hw, + [CLKID_VCLK] = &g12a_vclk.hw, + [CLKID_VCLK2] = &g12a_vclk2.hw, + [CLKID_VCLK_DIV1] = &g12a_vclk_div1.hw, + [CLKID_VCLK_DIV2_EN] = &g12a_vclk_div2_en.hw, + [CLKID_VCLK_DIV4_EN] = &g12a_vclk_div4_en.hw, + [CLKID_VCLK_DIV6_EN] = &g12a_vclk_div6_en.hw, + [CLKID_VCLK_DIV12_EN] = &g12a_vclk_div12_en.hw, + [CLKID_VCLK2_DIV1] = &g12a_vclk2_div1.hw, + [CLKID_VCLK2_DIV2_EN] = &g12a_vclk2_div2_en.hw, + [CLKID_VCLK2_DIV4_EN] = &g12a_vclk2_div4_en.hw, + [CLKID_VCLK2_DIV6_EN] = &g12a_vclk2_div6_en.hw, + [CLKID_VCLK2_DIV12_EN] = &g12a_vclk2_div12_en.hw, + [CLKID_VCLK_DIV2] = &g12a_vclk_div2.hw, + [CLKID_VCLK_DIV4] = &g12a_vclk_div4.hw, + [CLKID_VCLK_DIV6] = &g12a_vclk_div6.hw, + [CLKID_VCLK_DIV12] = &g12a_vclk_div12.hw, + [CLKID_VCLK2_DIV2] = &g12a_vclk2_div2.hw, + [CLKID_VCLK2_DIV4] = &g12a_vclk2_div4.hw, + [CLKID_VCLK2_DIV6] = &g12a_vclk2_div6.hw, + [CLKID_VCLK2_DIV12] = &g12a_vclk2_div12.hw, + [CLKID_CTS_ENCI_SEL] = &g12a_cts_enci_sel.hw, + [CLKID_CTS_ENCP_SEL] = &g12a_cts_encp_sel.hw, + [CLKID_CTS_VDAC_SEL] = &g12a_cts_vdac_sel.hw, + [CLKID_HDMI_TX_SEL] = &g12a_hdmi_tx_sel.hw, + [CLKID_CTS_ENCI] = &g12a_cts_enci.hw, + [CLKID_CTS_ENCP] = &g12a_cts_encp.hw, + [CLKID_CTS_VDAC] = &g12a_cts_vdac.hw, + [CLKID_HDMI_TX] = &g12a_hdmi_tx.hw, + [CLKID_HDMI_SEL] = &g12a_hdmi_sel.hw, + [CLKID_HDMI_DIV] = &g12a_hdmi_div.hw, + [CLKID_HDMI] = &g12a_hdmi.hw, + [CLKID_MALI_0_SEL] = &g12a_mali_0_sel.hw, + [CLKID_MALI_0_DIV] = &g12a_mali_0_div.hw, + [CLKID_MALI_0] = &g12a_mali_0.hw, + [CLKID_MALI_1_SEL] = &g12a_mali_1_sel.hw, + [CLKID_MALI_1_DIV] = &g12a_mali_1_div.hw, + [CLKID_MALI_1] = &g12a_mali_1.hw, + [CLKID_MALI] = &g12a_mali.hw, + [CLKID_MPLL_5OM_DIV] = &g12a_mpll_50m_div.hw, + [CLKID_MPLL_5OM] = &g12a_mpll_50m.hw, + [NR_CLKS] = NULL, + }, + .num = NR_CLKS, +}; + +/* Convenience table to populate regmap in .probe */ +static struct clk_regmap *const g12a_clk_regmaps[] = { + &g12a_clk81, + &g12a_dos, + &g12a_ddr, + &g12a_audio_locker, + &g12a_mipi_dsi_host, + &g12a_eth_phy, + &g12a_isa, + &g12a_pl301, + &g12a_periphs, + &g12a_spicc_0, + &g12a_i2c, + &g12a_sana, + &g12a_sd, + &g12a_rng0, + &g12a_uart0, + &g12a_spicc_1, + &g12a_hiu_reg, + &g12a_mipi_dsi_phy, + &g12a_assist_misc, + &g12a_emmc_a, + &g12a_emmc_b, + &g12a_emmc_c, + &g12a_audio_codec, + &g12a_audio, + &g12a_eth_core, + &g12a_demux, + &g12a_audio_ififo, + &g12a_adc, + &g12a_uart1, + &g12a_g2d, + &g12a_reset, + &g12a_pcie_comb, + &g12a_parser, + &g12a_usb_general, + &g12a_pcie_phy, + &g12a_ahb_arb0, + &g12a_ahb_data_bus, + &g12a_ahb_ctrl_bus, + &g12a_htx_hdcp22, + &g12a_htx_pclk, + &g12a_bt656, + &g12a_usb1_to_ddr, + &g12a_mmc_pclk, + &g12a_vpu_intr, + &g12a_gic, + &g12a_sd_emmc_a_clk0, + &g12a_sd_emmc_b_clk0, + &g12a_sd_emmc_c_clk0, + &g12a_mpeg_clk_div, + &g12a_sd_emmc_a_clk0_div, + &g12a_sd_emmc_b_clk0_div, + &g12a_sd_emmc_c_clk0_div, + &g12a_mpeg_clk_sel, + &g12a_sd_emmc_a_clk0_sel, + &g12a_sd_emmc_b_clk0_sel, + &g12a_sd_emmc_c_clk0_sel, + &g12a_mpll0, + &g12a_mpll1, + &g12a_mpll2, + &g12a_mpll3, + &g12a_mpll0_div, + &g12a_mpll1_div, + &g12a_mpll2_div, + &g12a_mpll3_div, + &g12a_fixed_pll, + &g12a_sys_pll, + &g12a_gp0_pll, + &g12a_hifi_pll, + &g12a_vclk2_venci0, + &g12a_vclk2_venci1, + &g12a_vclk2_vencp0, + &g12a_vclk2_vencp1, + &g12a_vclk2_venct0, + &g12a_vclk2_venct1, + &g12a_vclk2_other, + &g12a_vclk2_enci, + &g12a_vclk2_encp, + &g12a_dac_clk, + &g12a_aoclk_gate, + &g12a_iec958_gate, + &g12a_enc480p, + &g12a_rng1, + &g12a_vclk2_enct, + &g12a_vclk2_encl, + &g12a_vclk2_venclmmc, + &g12a_vclk2_vencl, + &g12a_vclk2_other1, + &g12a_fixed_pll_dco, + &g12a_sys_pll_dco, + &g12a_gp0_pll_dco, + &g12a_hifi_pll_dco, + &g12a_fclk_div2, + &g12a_fclk_div3, + &g12a_fclk_div4, + &g12a_fclk_div5, + &g12a_fclk_div7, + &g12a_fclk_div2p5, + &g12a_dma, + &g12a_efuse, + &g12a_rom_boot, + &g12a_reset_sec, + &g12a_sec_ahb_apb3, + &g12a_vpu_0_sel, + &g12a_vpu_0_div, + &g12a_vpu_0, + &g12a_vpu_1_sel, + &g12a_vpu_1_div, + &g12a_vpu_1, + &g12a_vpu, + &g12a_vapb_0_sel, + &g12a_vapb_0_div, + &g12a_vapb_0, + &g12a_vapb_1_sel, + &g12a_vapb_1_div, + &g12a_vapb_1, + &g12a_vapb_sel, + &g12a_vapb, + &g12a_hdmi_pll_dco, + &g12a_hdmi_pll_od, + &g12a_hdmi_pll_od2, + &g12a_hdmi_pll, + &g12a_vid_pll_div, + &g12a_vid_pll_sel, + &g12a_vid_pll, + &g12a_vclk_sel, + &g12a_vclk2_sel, + &g12a_vclk_input, + &g12a_vclk2_input, + &g12a_vclk_div, + &g12a_vclk2_div, + &g12a_vclk, + &g12a_vclk2, + &g12a_vclk_div1, + &g12a_vclk_div2_en, + &g12a_vclk_div4_en, + &g12a_vclk_div6_en, + &g12a_vclk_div12_en, + &g12a_vclk2_div1, + &g12a_vclk2_div2_en, + &g12a_vclk2_div4_en, + &g12a_vclk2_div6_en, + &g12a_vclk2_div12_en, + &g12a_cts_enci_sel, + &g12a_cts_encp_sel, + &g12a_cts_vdac_sel, + &g12a_hdmi_tx_sel, + &g12a_cts_enci, + &g12a_cts_encp, + &g12a_cts_vdac, + &g12a_hdmi_tx, + &g12a_hdmi_sel, + &g12a_hdmi_div, + &g12a_hdmi, + &g12a_mali_0_sel, + &g12a_mali_0_div, + &g12a_mali_0, + &g12a_mali_1_sel, + &g12a_mali_1_div, + &g12a_mali_1, + &g12a_mali, + &g12a_mpll_50m, +}; + +static const struct meson_eeclkc_data g12a_clkc_data = { + .regmap_clks = g12a_clk_regmaps, + .regmap_clk_num = ARRAY_SIZE(g12a_clk_regmaps), + .hw_onecell_data = &g12a_hw_onecell_data +}; + +static const struct of_device_id clkc_match_table[] = { + { .compatible = "amlogic,g12a-clkc", .data = &g12a_clkc_data }, + {} +}; + +static struct platform_driver g12a_driver = { + .probe = meson_eeclkc_probe, + .driver = { + .name = "g12a-clkc", + .of_match_table = clkc_match_table, + }, +}; + +builtin_platform_driver(g12a_driver); diff --git a/drivers/clk/meson/g12a.h b/drivers/clk/meson/g12a.h new file mode 100644 index 0000000000000..f399dfe1401cd --- /dev/null +++ b/drivers/clk/meson/g12a.h @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */ +/* + * Copyright (c) 2016 Amlogic, Inc. + * Author: Michael Turquette <mturquette@baylibre.com> + * + * Copyright (c) 2018 Amlogic, inc. + * Author: Qiufang Dai <qiufang.dai@amlogic.com> + * Author: Jian Hu <jian.hu@amlogic.com> + * + */ +#ifndef __G12A_H +#define __G12A_H + +/* + * Clock controller register offsets + * + * Register offsets from the data sheet must be multiplied by 4 before + * adding them to the base address to get the right value. + */ +#define HHI_MIPI_CNTL0 0x000 +#define HHI_MIPI_CNTL1 0x004 +#define HHI_MIPI_CNTL2 0x008 +#define HHI_MIPI_STS 0x00C +#define HHI_GP0_PLL_CNTL0 0x040 +#define HHI_GP0_PLL_CNTL1 0x044 +#define HHI_GP0_PLL_CNTL2 0x048 +#define HHI_GP0_PLL_CNTL3 0x04C +#define HHI_GP0_PLL_CNTL4 0x050 +#define HHI_GP0_PLL_CNTL5 0x054 +#define HHI_GP0_PLL_CNTL6 0x058 +#define HHI_GP0_PLL_STS 0x05C +#define HHI_PCIE_PLL_CNTL0 0x098 +#define HHI_PCIE_PLL_CNTL1 0x09C +#define HHI_PCIE_PLL_CNTL2 0x0A0 +#define HHI_PCIE_PLL_CNTL3 0x0A4 +#define HHI_PCIE_PLL_CNTL4 0x0A8 +#define HHI_PCIE_PLL_CNTL5 0x0AC +#define HHI_PCIE_PLL_STS 0x0B8 +#define HHI_HIFI_PLL_CNTL0 0x0D8 +#define HHI_HIFI_PLL_CNTL1 0x0DC +#define HHI_HIFI_PLL_CNTL2 0x0E0 +#define HHI_HIFI_PLL_CNTL3 0x0E4 +#define HHI_HIFI_PLL_CNTL4 0x0E8 +#define HHI_HIFI_PLL_CNTL5 0x0EC +#define HHI_HIFI_PLL_CNTL6 0x0F0 +#define HHI_VIID_CLK_DIV 0x128 +#define HHI_VIID_CLK_CNTL 0x12C +#define HHI_GCLK_MPEG0 0x140 +#define HHI_GCLK_MPEG1 0x144 +#define HHI_GCLK_MPEG2 0x148 +#define HHI_GCLK_OTHER 0x150 +#define HHI_GCLK_OTHER2 0x154 +#define HHI_VID_CLK_DIV 0x164 +#define HHI_MPEG_CLK_CNTL 0x174 +#define HHI_AUD_CLK_CNTL 0x178 +#define HHI_VID_CLK_CNTL 0x17c +#define HHI_TS_CLK_CNTL 0x190 +#define HHI_VID_CLK_CNTL2 0x194 +#define HHI_SYS_CPU_CLK_CNTL0 0x19c +#define HHI_VID_PLL_CLK_DIV 0x1A0 +#define HHI_MALI_CLK_CNTL 0x1b0 +#define HHI_VPU_CLKC_CNTL 0x1b4 +#define HHI_VPU_CLK_CNTL 0x1bC +#define HHI_HDMI_CLK_CNTL 0x1CC +#define HHI_VDEC_CLK_CNTL 0x1E0 +#define HHI_VDEC2_CLK_CNTL 0x1E4 +#define HHI_VDEC3_CLK_CNTL 0x1E8 +#define HHI_VDEC4_CLK_CNTL 0x1EC +#define HHI_HDCP22_CLK_CNTL 0x1F0 +#define HHI_VAPBCLK_CNTL 0x1F4 +#define HHI_VPU_CLKB_CNTL 0x20C +#define HHI_GEN_CLK_CNTL 0x228 +#define HHI_VDIN_MEAS_CLK_CNTL 0x250 +#define HHI_MIPIDSI_PHY_CLK_CNTL 0x254 +#define HHI_NAND_CLK_CNTL 0x25C +#define HHI_SD_EMMC_CLK_CNTL 0x264 +#define HHI_MPLL_CNTL0 0x278 +#define HHI_MPLL_CNTL1 0x27C +#define HHI_MPLL_CNTL2 0x280 +#define HHI_MPLL_CNTL3 0x284 +#define HHI_MPLL_CNTL4 0x288 +#define HHI_MPLL_CNTL5 0x28c +#define HHI_MPLL_CNTL6 0x290 +#define HHI_MPLL_CNTL7 0x294 +#define HHI_MPLL_CNTL8 0x298 +#define HHI_FIX_PLL_CNTL0 0x2A0 +#define HHI_FIX_PLL_CNTL1 0x2A4 +#define HHI_FIX_PLL_CNTL3 0x2AC +#define HHI_SYS_PLL_CNTL0 0x2f4 +#define HHI_SYS_PLL_CNTL1 0x2f8 +#define HHI_SYS_PLL_CNTL2 0x2fc +#define HHI_SYS_PLL_CNTL3 0x300 +#define HHI_SYS_PLL_CNTL4 0x304 +#define HHI_SYS_PLL_CNTL5 0x308 +#define HHI_SYS_PLL_CNTL6 0x30c +#define HHI_HDMI_PLL_CNTL0 0x320 +#define HHI_HDMI_PLL_CNTL1 0x324 +#define HHI_HDMI_PLL_CNTL2 0x328 +#define HHI_HDMI_PLL_CNTL3 0x32c +#define HHI_HDMI_PLL_CNTL4 0x330 +#define HHI_HDMI_PLL_CNTL5 0x334 +#define HHI_HDMI_PLL_CNTL6 0x338 +#define HHI_SPICC_CLK_CNTL 0x3dc + +/* + * CLKID index values + * + * These indices are entirely contrived and do not map onto the hardware. + * It has now been decided to expose everything by default in the DT header: + * include/dt-bindings/clock/g12a-clkc.h. Only the clocks ids we don't want + * to expose, such as the internal muxes and dividers of composite clocks, + * will remain defined here. + */ +#define CLKID_MPEG_SEL 8 +#define CLKID_MPEG_DIV 9 +#define CLKID_SD_EMMC_A_CLK0_SEL 63 +#define CLKID_SD_EMMC_A_CLK0_DIV 64 +#define CLKID_SD_EMMC_B_CLK0_SEL 65 +#define CLKID_SD_EMMC_B_CLK0_DIV 66 +#define CLKID_SD_EMMC_C_CLK0_SEL 67 +#define CLKID_SD_EMMC_C_CLK0_DIV 68 +#define CLKID_MPLL0_DIV 69 +#define CLKID_MPLL1_DIV 70 +#define CLKID_MPLL2_DIV 71 +#define CLKID_MPLL3_DIV 72 +#define CLKID_MPLL_PREDIV 73 +#define CLKID_FCLK_DIV2_DIV 75 +#define CLKID_FCLK_DIV3_DIV 76 +#define CLKID_FCLK_DIV4_DIV 77 +#define CLKID_FCLK_DIV5_DIV 78 +#define CLKID_FCLK_DIV7_DIV 79 +#define CLKID_FCLK_DIV2P5_DIV 100 +#define CLKID_FIXED_PLL_DCO 101 +#define CLKID_SYS_PLL_DCO 102 +#define CLKID_GP0_PLL_DCO 103 +#define CLKID_HIFI_PLL_DCO 104 +#define CLKID_VPU_0_DIV 111 +#define CLKID_VPU_1_DIV 114 +#define CLKID_VAPB_0_DIV 118 +#define CLKID_VAPB_1_DIV 121 +#define CLKID_HDMI_PLL_DCO 125 +#define CLKID_HDMI_PLL_OD 126 +#define CLKID_HDMI_PLL_OD2 127 +#define CLKID_VID_PLL_SEL 130 +#define CLKID_VID_PLL_DIV 131 +#define CLKID_VCLK_SEL 132 +#define CLKID_VCLK2_SEL 133 +#define CLKID_VCLK_INPUT 134 +#define CLKID_VCLK2_INPUT 135 +#define CLKID_VCLK_DIV 136 +#define CLKID_VCLK2_DIV 137 +#define CLKID_VCLK_DIV2_EN 140 +#define CLKID_VCLK_DIV4_EN 141 +#define CLKID_VCLK_DIV6_EN 142 +#define CLKID_VCLK_DIV12_EN 143 +#define CLKID_VCLK2_DIV2_EN 144 +#define CLKID_VCLK2_DIV4_EN 145 +#define CLKID_VCLK2_DIV6_EN 146 +#define CLKID_VCLK2_DIV12_EN 147 +#define CLKID_CTS_ENCI_SEL 158 +#define CLKID_CTS_ENCP_SEL 159 +#define CLKID_CTS_VDAC_SEL 160 +#define CLKID_HDMI_TX_SEL 161 +#define CLKID_HDMI_SEL 166 +#define CLKID_HDMI_DIV 167 +#define CLKID_MALI_0_DIV 170 +#define CLKID_MALI_1_DIV 173 +#define CLKID_MPLL_5OM_DIV 176 + +#define NR_CLKS 178 + +/* include the CLKIDs that have been made part of the DT binding */ +#include <dt-bindings/clock/g12a-clkc.h> + +#endif /* __G12A_H */ diff --git a/drivers/clk/meson/gxbb-aoclk-32k.c b/drivers/clk/meson/gxbb-aoclk-32k.c deleted file mode 100644 index 680467141a1d6..0000000000000 --- a/drivers/clk/meson/gxbb-aoclk-32k.c +++ /dev/null @@ -1,193 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Copyright (c) 2017 BayLibre, SAS. - * Author: Neil Armstrong <narmstrong@baylibre.com> - */ - -#include <linux/clk-provider.h> -#include <linux/bitfield.h> -#include <linux/regmap.h> -#include "gxbb-aoclk.h" - -/* - * The AO Domain embeds a dual/divider to generate a more precise - * 32,768KHz clock for low-power suspend mode and CEC. - * ______ ______ - * | | | | - * ______ | Div1 |-| Cnt1 | ______ - * | | /|______| |______|\ | | - * Xtal-->| Gate |---| ______ ______ X-X--| Gate |--> - * |______| | \| | | |/ | |______| - * | | Div2 |-| Cnt2 | | - * | |______| |______| | - * |_______________________| - * - * The dividing can be switched to single or dual, with a counter - * for each divider to set when the switching is done. - * The entire dividing mechanism can be also bypassed. - */ - -#define CLK_CNTL0_N1_MASK GENMASK(11, 0) -#define CLK_CNTL0_N2_MASK GENMASK(23, 12) -#define CLK_CNTL0_DUALDIV_EN BIT(28) -#define CLK_CNTL0_OUT_GATE_EN BIT(30) -#define CLK_CNTL0_IN_GATE_EN BIT(31) - -#define CLK_CNTL1_M1_MASK GENMASK(11, 0) -#define CLK_CNTL1_M2_MASK GENMASK(23, 12) -#define CLK_CNTL1_BYPASS_EN BIT(24) -#define CLK_CNTL1_SELECT_OSC BIT(27) - -#define PWR_CNTL_ALT_32K_SEL GENMASK(13, 10) - -struct cec_32k_freq_table { - unsigned long parent_rate; - unsigned long target_rate; - bool dualdiv; - unsigned int n1; - unsigned int n2; - unsigned int m1; - unsigned int m2; -}; - -static const struct cec_32k_freq_table aoclk_cec_32k_table[] = { - [0] = { - .parent_rate = 24000000, - .target_rate = 32768, - .dualdiv = true, - .n1 = 733, - .n2 = 732, - .m1 = 8, - .m2 = 11, - }, -}; - -/* - * If CLK_CNTL0_DUALDIV_EN == 0 - * - will use N1 divider only - * If CLK_CNTL0_DUALDIV_EN == 1 - * - hold M1 cycles of N1 divider then changes to N2 - * - hold M2 cycles of N2 divider then changes to N1 - * Then we can get more accurate division. - */ -static unsigned long aoclk_cec_32k_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct aoclk_cec_32k *cec_32k = to_aoclk_cec_32k(hw); - unsigned long n1; - u32 reg0, reg1; - - regmap_read(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL0, ®0); - regmap_read(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL1, ®1); - - if (reg1 & CLK_CNTL1_BYPASS_EN) - return parent_rate; - - if (reg0 & CLK_CNTL0_DUALDIV_EN) { - unsigned long n2, m1, m2, f1, f2, p1, p2; - - n1 = FIELD_GET(CLK_CNTL0_N1_MASK, reg0) + 1; - n2 = FIELD_GET(CLK_CNTL0_N2_MASK, reg0) + 1; - - m1 = FIELD_GET(CLK_CNTL1_M1_MASK, reg1) + 1; - m2 = FIELD_GET(CLK_CNTL1_M2_MASK, reg1) + 1; - - f1 = DIV_ROUND_CLOSEST(parent_rate, n1); - f2 = DIV_ROUND_CLOSEST(parent_rate, n2); - - p1 = DIV_ROUND_CLOSEST(100000000 * m1, f1 * (m1 + m2)); - p2 = DIV_ROUND_CLOSEST(100000000 * m2, f2 * (m1 + m2)); - - return DIV_ROUND_UP(100000000, p1 + p2); - } - - n1 = FIELD_GET(CLK_CNTL0_N1_MASK, reg0) + 1; - - return DIV_ROUND_CLOSEST(parent_rate, n1); -} - -static const struct cec_32k_freq_table *find_cec_32k_freq(unsigned long rate, - unsigned long prate) -{ - int i; - - for (i = 0 ; i < ARRAY_SIZE(aoclk_cec_32k_table) ; ++i) - if (aoclk_cec_32k_table[i].parent_rate == prate && - aoclk_cec_32k_table[i].target_rate == rate) - return &aoclk_cec_32k_table[i]; - - return NULL; -} - -static long aoclk_cec_32k_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) -{ - const struct cec_32k_freq_table *freq = find_cec_32k_freq(rate, - *prate); - - /* If invalid return first one */ - if (!freq) - return aoclk_cec_32k_table[0].target_rate; - - return freq->target_rate; -} - -/* - * From the Amlogic init procedure, the IN and OUT gates needs to be handled - * in the init procedure to avoid any glitches. - */ - -static int aoclk_cec_32k_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - const struct cec_32k_freq_table *freq = find_cec_32k_freq(rate, - parent_rate); - struct aoclk_cec_32k *cec_32k = to_aoclk_cec_32k(hw); - u32 reg = 0; - - if (!freq) - return -EINVAL; - - /* Disable clock */ - regmap_update_bits(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL0, - CLK_CNTL0_IN_GATE_EN | CLK_CNTL0_OUT_GATE_EN, 0); - - reg = FIELD_PREP(CLK_CNTL0_N1_MASK, freq->n1 - 1); - if (freq->dualdiv) - reg |= CLK_CNTL0_DUALDIV_EN | - FIELD_PREP(CLK_CNTL0_N2_MASK, freq->n2 - 1); - - regmap_write(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL0, reg); - - reg = FIELD_PREP(CLK_CNTL1_M1_MASK, freq->m1 - 1); - if (freq->dualdiv) - reg |= FIELD_PREP(CLK_CNTL1_M2_MASK, freq->m2 - 1); - - regmap_write(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL1, reg); - - /* Enable clock */ - regmap_update_bits(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL0, - CLK_CNTL0_IN_GATE_EN, CLK_CNTL0_IN_GATE_EN); - - udelay(200); - - regmap_update_bits(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL0, - CLK_CNTL0_OUT_GATE_EN, CLK_CNTL0_OUT_GATE_EN); - - regmap_update_bits(cec_32k->regmap, AO_CRT_CLK_CNTL1, - CLK_CNTL1_SELECT_OSC, CLK_CNTL1_SELECT_OSC); - - /* Select 32k from XTAL */ - regmap_update_bits(cec_32k->regmap, - AO_RTI_PWR_CNTL_REG0, - PWR_CNTL_ALT_32K_SEL, - FIELD_PREP(PWR_CNTL_ALT_32K_SEL, 4)); - - return 0; -} - -const struct clk_ops meson_aoclk_cec_32k_ops = { - .recalc_rate = aoclk_cec_32k_recalc_rate, - .round_rate = aoclk_cec_32k_round_rate, - .set_rate = aoclk_cec_32k_set_rate, -}; diff --git a/drivers/clk/meson/gxbb-aoclk.c b/drivers/clk/meson/gxbb-aoclk.c index 42ed61d3c3fba..449f6ac189d86 100644 --- a/drivers/clk/meson/gxbb-aoclk.c +++ b/drivers/clk/meson/gxbb-aoclk.c @@ -5,10 +5,23 @@ */ #include <linux/platform_device.h> #include <linux/mfd/syscon.h> -#include "clk-regmap.h" #include "meson-aoclk.h" #include "gxbb-aoclk.h" +#include "clk-regmap.h" +#include "clk-dualdiv.h" + +#define IN_PREFIX "ao-in-" + +/* AO Configuration Clock registers offsets */ +#define AO_RTI_PWR_CNTL_REG1 0x0c +#define AO_RTI_PWR_CNTL_REG0 0x10 +#define AO_RTI_GEN_CNTL_REG0 0x40 +#define AO_OSCIN_CNTL 0x58 +#define AO_CRT_CLK_CNTL1 0x68 +#define AO_RTC_ALT_CLK_CNTL0 0x94 +#define AO_RTC_ALT_CLK_CNTL1 0x98 + #define GXBB_AO_GATE(_name, _bit) \ static struct clk_regmap _name##_ao = { \ .data = &(struct clk_regmap_gate_data) { \ @@ -18,7 +31,7 @@ static struct clk_regmap _name##_ao = { \ .hw.init = &(struct clk_init_data) { \ .name = #_name "_ao", \ .ops = &clk_regmap_gate_ops, \ - .parent_names = (const char *[]){ "clk81" }, \ + .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk" }, \ .num_parents = 1, \ .flags = CLK_IGNORE_UNUSED, \ }, \ @@ -31,13 +44,174 @@ GXBB_AO_GATE(uart1, 3); GXBB_AO_GATE(uart2, 5); GXBB_AO_GATE(ir_blaster, 6); -static struct aoclk_cec_32k cec_32k_ao = { - .hw.init = &(struct clk_init_data) { - .name = "cec_32k_ao", - .ops = &meson_aoclk_cec_32k_ops, - .parent_names = (const char *[]){ "xtal" }, +static struct clk_regmap ao_cts_oscin = { + .data = &(struct clk_regmap_gate_data){ + .offset = AO_RTI_PWR_CNTL_REG0, + .bit_idx = 6, + }, + .hw.init = &(struct clk_init_data){ + .name = "ao_cts_oscin", + .ops = &clk_regmap_gate_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap ao_32k_pre = { + .data = &(struct clk_regmap_gate_data){ + .offset = AO_RTC_ALT_CLK_CNTL0, + .bit_idx = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "ao_32k_pre", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "ao_cts_oscin" }, + .num_parents = 1, + }, +}; + +static const struct meson_clk_dualdiv_param gxbb_32k_div_table[] = { + { + .dual = 1, + .n1 = 733, + .m1 = 8, + .n2 = 732, + .m2 = 11, + }, {} +}; + +static struct clk_regmap ao_32k_div = { + .data = &(struct meson_clk_dualdiv_data){ + .n1 = { + .reg_off = AO_RTC_ALT_CLK_CNTL0, + .shift = 0, + .width = 12, + }, + .n2 = { + .reg_off = AO_RTC_ALT_CLK_CNTL0, + .shift = 12, + .width = 12, + }, + .m1 = { + .reg_off = AO_RTC_ALT_CLK_CNTL1, + .shift = 0, + .width = 12, + }, + .m2 = { + .reg_off = AO_RTC_ALT_CLK_CNTL1, + .shift = 12, + .width = 12, + }, + .dual = { + .reg_off = AO_RTC_ALT_CLK_CNTL0, + .shift = 28, + .width = 1, + }, + .table = gxbb_32k_div_table, + }, + .hw.init = &(struct clk_init_data){ + .name = "ao_32k_div", + .ops = &meson_clk_dualdiv_ops, + .parent_names = (const char *[]){ "ao_32k_pre" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap ao_32k_sel = { + .data = &(struct clk_regmap_mux_data) { + .offset = AO_RTC_ALT_CLK_CNTL1, + .mask = 0x1, + .shift = 24, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "ao_32k_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = (const char *[]){ "ao_32k_div", + "ao_32k_pre" }, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap ao_32k = { + .data = &(struct clk_regmap_gate_data){ + .offset = AO_RTC_ALT_CLK_CNTL0, + .bit_idx = 30, + }, + .hw.init = &(struct clk_init_data){ + .name = "ao_32k", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "ao_32k_sel" }, .num_parents = 1, - .flags = CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap ao_cts_rtc_oscin = { + .data = &(struct clk_regmap_mux_data) { + .offset = AO_RTI_PWR_CNTL_REG0, + .mask = 0x7, + .shift = 10, + .table = (u32[]){ 1, 2, 3, 4 }, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "ao_cts_rtc_oscin", + .ops = &clk_regmap_mux_ops, + .parent_names = (const char *[]){ IN_PREFIX "ext-32k-0", + IN_PREFIX "ext-32k-1", + IN_PREFIX "ext-32k-2", + "ao_32k" }, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap ao_clk81 = { + .data = &(struct clk_regmap_mux_data) { + .offset = AO_RTI_PWR_CNTL_REG0, + .mask = 0x1, + .shift = 0, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "ao_clk81", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk", + "ao_cts_rtc_oscin" }, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap ao_cts_cec = { + .data = &(struct clk_regmap_mux_data) { + .offset = AO_CRT_CLK_CNTL1, + .mask = 0x1, + .shift = 27, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "ao_cts_cec", + .ops = &clk_regmap_mux_ops, + /* + * FIXME: The 'fixme' parent obviously does not exist. + * + * ATM, CCF won't call get_parent() if num_parents is 1. It + * does not allow NULL as a parent name either. + * + * On this particular mux, we only know the input #1 parent + * but, on boot, unknown input #0 is set, so it is critical + * to call .get_parent() on it + * + * Until CCF gets fixed, adding this fake parent that won't + * ever be registered should work around the problem + */ + .parent_names = (const char *[]){ "fixme", + "ao_cts_rtc_oscin" }, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -50,13 +224,21 @@ static const unsigned int gxbb_aoclk_reset[] = { [RESET_AO_IR_BLASTER] = 23, }; -static struct clk_regmap *gxbb_aoclk_gate[] = { - [CLKID_AO_REMOTE] = &remote_ao, - [CLKID_AO_I2C_MASTER] = &i2c_master_ao, - [CLKID_AO_I2C_SLAVE] = &i2c_slave_ao, - [CLKID_AO_UART1] = &uart1_ao, - [CLKID_AO_UART2] = &uart2_ao, - [CLKID_AO_IR_BLASTER] = &ir_blaster_ao, +static struct clk_regmap *gxbb_aoclk[] = { + &remote_ao, + &i2c_master_ao, + &i2c_slave_ao, + &uart1_ao, + &uart2_ao, + &ir_blaster_ao, + &ao_cts_oscin, + &ao_32k_pre, + &ao_32k_div, + &ao_32k_sel, + &ao_32k, + &ao_cts_rtc_oscin, + &ao_clk81, + &ao_cts_cec, }; static const struct clk_hw_onecell_data gxbb_aoclk_onecell_data = { @@ -67,52 +249,38 @@ static const struct clk_hw_onecell_data gxbb_aoclk_onecell_data = { [CLKID_AO_UART1] = &uart1_ao.hw, [CLKID_AO_UART2] = &uart2_ao.hw, [CLKID_AO_IR_BLASTER] = &ir_blaster_ao.hw, - [CLKID_AO_CEC_32K] = &cec_32k_ao.hw, + [CLKID_AO_CEC_32K] = &ao_cts_cec.hw, + [CLKID_AO_CTS_OSCIN] = &ao_cts_oscin.hw, + [CLKID_AO_32K_PRE] = &ao_32k_pre.hw, + [CLKID_AO_32K_DIV] = &ao_32k_div.hw, + [CLKID_AO_32K_SEL] = &ao_32k_sel.hw, + [CLKID_AO_32K] = &ao_32k.hw, + [CLKID_AO_CTS_RTC_OSCIN] = &ao_cts_rtc_oscin.hw, + [CLKID_AO_CLK81] = &ao_clk81.hw, }, .num = NR_CLKS, }; -static int gxbb_register_cec_ao_32k(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct regmap *regmap; - int ret; - - regmap = syscon_node_to_regmap(of_get_parent(dev->of_node)); - if (IS_ERR(regmap)) { - dev_err(dev, "failed to get regmap\n"); - return PTR_ERR(regmap); - } - - /* Specific clocks */ - cec_32k_ao.regmap = regmap; - ret = devm_clk_hw_register(dev, &cec_32k_ao.hw); - if (ret) { - dev_err(&pdev->dev, "clk cec_32k_ao register failed.\n"); - return ret; - } - - return 0; -} +static const struct meson_aoclk_input gxbb_aoclk_inputs[] = { + { .name = "xtal", .required = true, }, + { .name = "mpeg-clk", .required = true, }, + {. name = "ext-32k-0", .required = false, }, + {. name = "ext-32k-1", .required = false, }, + {. name = "ext-32k-2", .required = false, }, +}; static const struct meson_aoclk_data gxbb_aoclkc_data = { .reset_reg = AO_RTI_GEN_CNTL_REG0, .num_reset = ARRAY_SIZE(gxbb_aoclk_reset), .reset = gxbb_aoclk_reset, - .num_clks = ARRAY_SIZE(gxbb_aoclk_gate), - .clks = gxbb_aoclk_gate, + .num_clks = ARRAY_SIZE(gxbb_aoclk), + .clks = gxbb_aoclk, .hw_data = &gxbb_aoclk_onecell_data, + .inputs = gxbb_aoclk_inputs, + .num_inputs = ARRAY_SIZE(gxbb_aoclk_inputs), + .input_prefix = IN_PREFIX, }; -static int gxbb_aoclkc_probe(struct platform_device *pdev) -{ - int ret = gxbb_register_cec_ao_32k(pdev); - if (ret) - return ret; - - return meson_aoclkc_probe(pdev); -} - static const struct of_device_id gxbb_aoclkc_match_table[] = { { .compatible = "amlogic,meson-gx-aoclkc", @@ -122,7 +290,7 @@ static const struct of_device_id gxbb_aoclkc_match_table[] = { }; static struct platform_driver gxbb_aoclkc_driver = { - .probe = gxbb_aoclkc_probe, + .probe = meson_aoclkc_probe, .driver = { .name = "gxbb-aoclkc", .of_match_table = gxbb_aoclkc_match_table, diff --git a/drivers/clk/meson/gxbb-aoclk.h b/drivers/clk/meson/gxbb-aoclk.h index c514493d989a6..1db16f9b37d47 100644 --- a/drivers/clk/meson/gxbb-aoclk.h +++ b/drivers/clk/meson/gxbb-aoclk.h @@ -7,25 +7,7 @@ #ifndef __GXBB_AOCLKC_H #define __GXBB_AOCLKC_H -#define NR_CLKS 7 - -/* AO Configuration Clock registers offsets */ -#define AO_RTI_PWR_CNTL_REG1 0x0c -#define AO_RTI_PWR_CNTL_REG0 0x10 -#define AO_RTI_GEN_CNTL_REG0 0x40 -#define AO_OSCIN_CNTL 0x58 -#define AO_CRT_CLK_CNTL1 0x68 -#define AO_RTC_ALT_CLK_CNTL0 0x94 -#define AO_RTC_ALT_CLK_CNTL1 0x98 - -struct aoclk_cec_32k { - struct clk_hw hw; - struct regmap *regmap; -}; - -#define to_aoclk_cec_32k(_hw) container_of(_hw, struct aoclk_cec_32k, hw) - -extern const struct clk_ops meson_aoclk_cec_32k_ops; +#define NR_CLKS 14 #include <dt-bindings/clock/gxbb-aoclkc.h> #include <dt-bindings/reset/gxbb-aoclkc.h> diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c index 65f2599e52434..04df2e208ed6e 100644 --- a/drivers/clk/meson/gxbb.c +++ b/drivers/clk/meson/gxbb.c @@ -4,17 +4,20 @@ * Michael Turquette <mturquette@baylibre.com> */ -#include <linux/clk.h> #include <linux/clk-provider.h> #include <linux/init.h> #include <linux/of_device.h> -#include <linux/mfd/syscon.h> #include <linux/platform_device.h> -#include <linux/regmap.h> -#include "clkc.h" #include "gxbb.h" +#include "clk-input.h" #include "clk-regmap.h" +#include "clk-pll.h" +#include "clk-mpll.h" +#include "meson-eeclk.h" +#include "vid-pll-div.h" + +#define IN_PREFIX "ee-in-" static DEFINE_SPINLOCK(meson_clk_lock); @@ -118,7 +121,7 @@ static struct clk_regmap gxbb_fixed_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "fixed_pll_dco", .ops = &meson_clk_pll_ro_ops, - .parent_names = (const char *[]){ "xtal" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, .num_parents = 1, }, }; @@ -148,7 +151,7 @@ static struct clk_fixed_factor gxbb_hdmi_pll_pre_mult = { .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_pre_mult", .ops = &clk_fixed_factor_ops, - .parent_names = (const char *[]){ "xtal" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, .num_parents = 1, }, }; @@ -241,7 +244,7 @@ static struct clk_regmap gxl_hdmi_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_dco", .ops = &meson_clk_pll_ro_ops, - .parent_names = (const char *[]){ "xtal" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, .num_parents = 1, /* * Display directly handle hdmi pll registers ATM, we need @@ -378,7 +381,7 @@ static struct clk_regmap gxbb_sys_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "sys_pll_dco", .ops = &meson_clk_pll_ro_ops, - .parent_names = (const char *[]){ "xtal" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, .num_parents = 1, }, }; @@ -439,7 +442,7 @@ static struct clk_regmap gxbb_gp0_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "gp0_pll_dco", .ops = &meson_clk_pll_ops, - .parent_names = (const char *[]){ "xtal" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, .num_parents = 1, }, }; @@ -491,7 +494,7 @@ static struct clk_regmap gxl_gp0_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "gp0_pll_dco", .ops = &meson_clk_pll_ops, - .parent_names = (const char *[]){ "xtal" }, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, .num_parents = 1, }, }; @@ -789,7 +792,7 @@ static struct clk_regmap gxbb_mpll2 = { static u32 mux_table_clk81[] = { 0, 2, 3, 4, 5, 6, 7 }; static const char * const clk81_parent_names[] = { - "xtal", "fclk_div7", "mpll1", "mpll2", "fclk_div4", + IN_PREFIX "xtal", "fclk_div7", "mpll1", "mpll2", "fclk_div4", "fclk_div3", "fclk_div5" }; @@ -852,7 +855,7 @@ static struct clk_regmap gxbb_sar_adc_clk_sel = { .name = "sar_adc_clk_sel", .ops = &clk_regmap_mux_ops, /* NOTE: The datasheet doesn't list the parents for bit 10 */ - .parent_names = (const char *[]){ "xtal", "clk81", }, + .parent_names = (const char *[]){ IN_PREFIX "xtal", "clk81", }, .num_parents = 2, }, }; @@ -891,7 +894,7 @@ static struct clk_regmap gxbb_sar_adc_clk = { */ static const char * const gxbb_mali_0_1_parent_names[] = { - "xtal", "gp0_pll", "mpll2", "mpll1", "fclk_div7", + IN_PREFIX "xtal", "gp0_pll", "mpll2", "mpll1", "fclk_div7", "fclk_div4", "fclk_div3", "fclk_div5" }; @@ -1153,7 +1156,7 @@ static struct clk_regmap gxbb_32k_clk = { }; static const char * const gxbb_32k_clk_parent_names[] = { - "xtal", "cts_slow_oscin", "fclk_div3", "fclk_div5" + IN_PREFIX "xtal", "cts_slow_oscin", "fclk_div3", "fclk_div5" }; static struct clk_regmap gxbb_32k_clk_sel = { @@ -1172,7 +1175,7 @@ static struct clk_regmap gxbb_32k_clk_sel = { }; static const char * const gxbb_sd_emmc_clk0_parent_names[] = { - "xtal", "fclk_div2", "fclk_div3", "fclk_div5", "fclk_div7", + IN_PREFIX "xtal", "fclk_div2", "fclk_div3", "fclk_div5", "fclk_div7", /* * Following these parent clocks, we should also have had mpll2, mpll3 @@ -2138,7 +2141,7 @@ static struct clk_regmap gxbb_hdmi_tx = { /* HDMI Clocks */ static const char * const gxbb_hdmi_parent_names[] = { - "xtal", "fclk_div4", "fclk_div3", "fclk_div5" + IN_PREFIX "xtal", "fclk_div4", "fclk_div3", "fclk_div5" }; static struct clk_regmap gxbb_hdmi_sel = { @@ -2285,7 +2288,7 @@ static struct clk_regmap gxbb_vdec_hevc = { static u32 mux_table_gen_clk[] = { 0, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, }; static const char * const gen_clk_parent_names[] = { - "xtal", "vdec_1", "vdec_hevc", "mpll0", "mpll1", "mpll2", + IN_PREFIX "xtal", "vdec_1", "vdec_hevc", "mpll0", "mpll1", "mpll2", "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7", "gp0_pll", }; @@ -2854,6 +2857,192 @@ static struct clk_hw_onecell_data gxl_hw_onecell_data = { }; static struct clk_regmap *const gxbb_clk_regmaps[] = { + &gxbb_clk81, + &gxbb_ddr, + &gxbb_dos, + &gxbb_isa, + &gxbb_pl301, + &gxbb_periphs, + &gxbb_spicc, + &gxbb_i2c, + &gxbb_sar_adc, + &gxbb_smart_card, + &gxbb_rng0, + &gxbb_uart0, + &gxbb_sdhc, + &gxbb_stream, + &gxbb_async_fifo, + &gxbb_sdio, + &gxbb_abuf, + &gxbb_hiu_iface, + &gxbb_assist_misc, + &gxbb_spi, + &gxbb_i2s_spdif, + &gxbb_eth, + &gxbb_demux, + &gxbb_aiu_glue, + &gxbb_iec958, + &gxbb_i2s_out, + &gxbb_amclk, + &gxbb_aififo2, + &gxbb_mixer, + &gxbb_mixer_iface, + &gxbb_adc, + &gxbb_blkmv, + &gxbb_aiu, + &gxbb_uart1, + &gxbb_g2d, + &gxbb_usb0, + &gxbb_usb1, + &gxbb_reset, + &gxbb_nand, + &gxbb_dos_parser, + &gxbb_usb, + &gxbb_vdin1, + &gxbb_ahb_arb0, + &gxbb_efuse, + &gxbb_boot_rom, + &gxbb_ahb_data_bus, + &gxbb_ahb_ctrl_bus, + &gxbb_hdmi_intr_sync, + &gxbb_hdmi_pclk, + &gxbb_usb1_ddr_bridge, + &gxbb_usb0_ddr_bridge, + &gxbb_mmc_pclk, + &gxbb_dvin, + &gxbb_uart2, + &gxbb_sana, + &gxbb_vpu_intr, + &gxbb_sec_ahb_ahb3_bridge, + &gxbb_clk81_a53, + &gxbb_vclk2_venci0, + &gxbb_vclk2_venci1, + &gxbb_vclk2_vencp0, + &gxbb_vclk2_vencp1, + &gxbb_gclk_venci_int0, + &gxbb_gclk_vencp_int, + &gxbb_dac_clk, + &gxbb_aoclk_gate, + &gxbb_iec958_gate, + &gxbb_enc480p, + &gxbb_rng1, + &gxbb_gclk_venci_int1, + &gxbb_vclk2_venclmcc, + &gxbb_vclk2_vencl, + &gxbb_vclk_other, + &gxbb_edp, + &gxbb_ao_media_cpu, + &gxbb_ao_ahb_sram, + &gxbb_ao_ahb_bus, + &gxbb_ao_iface, + &gxbb_ao_i2c, + &gxbb_emmc_a, + &gxbb_emmc_b, + &gxbb_emmc_c, + &gxbb_sar_adc_clk, + &gxbb_mali_0, + &gxbb_mali_1, + &gxbb_cts_amclk, + &gxbb_cts_mclk_i958, + &gxbb_32k_clk, + &gxbb_sd_emmc_a_clk0, + &gxbb_sd_emmc_b_clk0, + &gxbb_sd_emmc_c_clk0, + &gxbb_vpu_0, + &gxbb_vpu_1, + &gxbb_vapb_0, + &gxbb_vapb_1, + &gxbb_vapb, + &gxbb_mpeg_clk_div, + &gxbb_sar_adc_clk_div, + &gxbb_mali_0_div, + &gxbb_mali_1_div, + &gxbb_cts_mclk_i958_div, + &gxbb_32k_clk_div, + &gxbb_sd_emmc_a_clk0_div, + &gxbb_sd_emmc_b_clk0_div, + &gxbb_sd_emmc_c_clk0_div, + &gxbb_vpu_0_div, + &gxbb_vpu_1_div, + &gxbb_vapb_0_div, + &gxbb_vapb_1_div, + &gxbb_mpeg_clk_sel, + &gxbb_sar_adc_clk_sel, + &gxbb_mali_0_sel, + &gxbb_mali_1_sel, + &gxbb_mali, + &gxbb_cts_amclk_sel, + &gxbb_cts_mclk_i958_sel, + &gxbb_cts_i958, + &gxbb_32k_clk_sel, + &gxbb_sd_emmc_a_clk0_sel, + &gxbb_sd_emmc_b_clk0_sel, + &gxbb_sd_emmc_c_clk0_sel, + &gxbb_vpu_0_sel, + &gxbb_vpu_1_sel, + &gxbb_vpu, + &gxbb_vapb_0_sel, + &gxbb_vapb_1_sel, + &gxbb_vapb_sel, + &gxbb_mpll0, + &gxbb_mpll1, + &gxbb_mpll2, + &gxbb_mpll0_div, + &gxbb_mpll1_div, + &gxbb_mpll2_div, + &gxbb_cts_amclk_div, + &gxbb_fixed_pll, + &gxbb_sys_pll, + &gxbb_mpll_prediv, + &gxbb_fclk_div2, + &gxbb_fclk_div3, + &gxbb_fclk_div4, + &gxbb_fclk_div5, + &gxbb_fclk_div7, + &gxbb_vdec_1_sel, + &gxbb_vdec_1_div, + &gxbb_vdec_1, + &gxbb_vdec_hevc_sel, + &gxbb_vdec_hevc_div, + &gxbb_vdec_hevc, + &gxbb_gen_clk_sel, + &gxbb_gen_clk_div, + &gxbb_gen_clk, + &gxbb_fixed_pll_dco, + &gxbb_sys_pll_dco, + &gxbb_gp0_pll, + &gxbb_vid_pll, + &gxbb_vid_pll_sel, + &gxbb_vid_pll_div, + &gxbb_vclk, + &gxbb_vclk_sel, + &gxbb_vclk_div, + &gxbb_vclk_input, + &gxbb_vclk_div1, + &gxbb_vclk_div2_en, + &gxbb_vclk_div4_en, + &gxbb_vclk_div6_en, + &gxbb_vclk_div12_en, + &gxbb_vclk2, + &gxbb_vclk2_sel, + &gxbb_vclk2_div, + &gxbb_vclk2_input, + &gxbb_vclk2_div1, + &gxbb_vclk2_div2_en, + &gxbb_vclk2_div4_en, + &gxbb_vclk2_div6_en, + &gxbb_vclk2_div12_en, + &gxbb_cts_enci, + &gxbb_cts_enci_sel, + &gxbb_cts_encp, + &gxbb_cts_encp_sel, + &gxbb_cts_vdac, + &gxbb_cts_vdac_sel, + &gxbb_hdmi_tx, + &gxbb_hdmi_tx_sel, + &gxbb_hdmi_sel, + &gxbb_hdmi_div, + &gxbb_hdmi, &gxbb_gp0_pll_dco, &gxbb_hdmi_pll, &gxbb_hdmi_pll_od, @@ -2862,14 +3051,6 @@ static struct clk_regmap *const gxbb_clk_regmaps[] = { }; static struct clk_regmap *const gxl_clk_regmaps[] = { - &gxl_gp0_pll_dco, - &gxl_hdmi_pll, - &gxl_hdmi_pll_od, - &gxl_hdmi_pll_od2, - &gxl_hdmi_pll_dco, -}; - -static struct clk_regmap *const gx_clk_regmaps[] = { &gxbb_clk81, &gxbb_ddr, &gxbb_dos, @@ -3056,23 +3237,22 @@ static struct clk_regmap *const gx_clk_regmaps[] = { &gxbb_hdmi_sel, &gxbb_hdmi_div, &gxbb_hdmi, + &gxl_gp0_pll_dco, + &gxl_hdmi_pll, + &gxl_hdmi_pll_od, + &gxl_hdmi_pll_od2, + &gxl_hdmi_pll_dco, }; -struct clkc_data { - struct clk_regmap *const *regmap_clks; - unsigned int regmap_clks_count; - struct clk_hw_onecell_data *hw_onecell_data; -}; - -static const struct clkc_data gxbb_clkc_data = { +static const struct meson_eeclkc_data gxbb_clkc_data = { .regmap_clks = gxbb_clk_regmaps, - .regmap_clks_count = ARRAY_SIZE(gxbb_clk_regmaps), + .regmap_clk_num = ARRAY_SIZE(gxbb_clk_regmaps), .hw_onecell_data = &gxbb_hw_onecell_data, }; -static const struct clkc_data gxl_clkc_data = { +static const struct meson_eeclkc_data gxl_clkc_data = { .regmap_clks = gxl_clk_regmaps, - .regmap_clks_count = ARRAY_SIZE(gxl_clk_regmaps), + .regmap_clk_num = ARRAY_SIZE(gxl_clk_regmaps), .hw_onecell_data = &gxl_hw_onecell_data, }; @@ -3082,52 +3262,8 @@ static const struct of_device_id clkc_match_table[] = { {}, }; -static int gxbb_clkc_probe(struct platform_device *pdev) -{ - const struct clkc_data *clkc_data; - struct regmap *map; - int ret, i; - struct device *dev = &pdev->dev; - - clkc_data = of_device_get_match_data(dev); - if (!clkc_data) - return -EINVAL; - - /* Get the hhi system controller node if available */ - map = syscon_node_to_regmap(of_get_parent(dev->of_node)); - if (IS_ERR(map)) { - dev_err(dev, "failed to get HHI regmap\n"); - return PTR_ERR(map); - } - - /* Populate regmap for the common regmap backed clocks */ - for (i = 0; i < ARRAY_SIZE(gx_clk_regmaps); i++) - gx_clk_regmaps[i]->map = map; - - /* Populate regmap for soc specific clocks */ - for (i = 0; i < clkc_data->regmap_clks_count; i++) - clkc_data->regmap_clks[i]->map = map; - - /* Register all clks */ - for (i = 0; i < clkc_data->hw_onecell_data->num; i++) { - /* array might be sparse */ - if (!clkc_data->hw_onecell_data->hws[i]) - continue; - - ret = devm_clk_hw_register(dev, - clkc_data->hw_onecell_data->hws[i]); - if (ret) { - dev_err(dev, "Clock registration failed\n"); - return ret; - } - } - - return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, - clkc_data->hw_onecell_data); -} - static struct platform_driver gxbb_driver = { - .probe = gxbb_clkc_probe, + .probe = meson_eeclkc_probe, .driver = { .name = "gxbb-clkc", .of_match_table = clkc_match_table, diff --git a/drivers/clk/meson/meson-aoclk.c b/drivers/clk/meson/meson-aoclk.c index f965845917e34..b67951909e049 100644 --- a/drivers/clk/meson/meson-aoclk.c +++ b/drivers/clk/meson/meson-aoclk.c @@ -14,9 +14,11 @@ #include <linux/reset-controller.h> #include <linux/mfd/syscon.h> #include <linux/of_device.h> -#include "clk-regmap.h" +#include <linux/slab.h> #include "meson-aoclk.h" +#include "clk-input.h" + static int meson_aoclk_do_reset(struct reset_controller_dev *rcdev, unsigned long id) { @@ -31,6 +33,37 @@ static const struct reset_control_ops meson_aoclk_reset_ops = { .reset = meson_aoclk_do_reset, }; +static int meson_aoclkc_register_inputs(struct device *dev, + struct meson_aoclk_data *data) +{ + struct clk_hw *hw; + char *str; + int i; + + for (i = 0; i < data->num_inputs; i++) { + const struct meson_aoclk_input *in = &data->inputs[i]; + + str = kasprintf(GFP_KERNEL, "%s%s", data->input_prefix, + in->name); + if (!str) + return -ENOMEM; + + hw = meson_clk_hw_register_input(dev, in->name, str, 0); + kfree(str); + + if (IS_ERR(hw)) { + if (!in->required && PTR_ERR(hw) == -ENOENT) + continue; + else if (PTR_ERR(hw) != -EPROBE_DEFER) + dev_err(dev, "failed to register input %s\n", + in->name); + return PTR_ERR(hw); + } + } + + return 0; +} + int meson_aoclkc_probe(struct platform_device *pdev) { struct meson_aoclk_reset_controller *rstc; @@ -53,6 +86,10 @@ int meson_aoclkc_probe(struct platform_device *pdev) return PTR_ERR(regmap); } + ret = meson_aoclkc_register_inputs(dev, data); + if (ret) + return ret; + /* Reset Controller */ rstc->data = data; rstc->regmap = regmap; @@ -65,15 +102,20 @@ int meson_aoclkc_probe(struct platform_device *pdev) return ret; } - /* - * Populate regmap and register all clks - */ - for (clkid = 0; clkid < data->num_clks; clkid++) { + /* Populate regmap */ + for (clkid = 0; clkid < data->num_clks; clkid++) data->clks[clkid]->map = regmap; + /* Register all clks */ + for (clkid = 0; clkid < data->hw_data->num; clkid++) { + if (!data->hw_data->hws[clkid]) + continue; + ret = devm_clk_hw_register(dev, data->hw_data->hws[clkid]); - if (ret) + if (ret) { + dev_err(dev, "Clock registration failed\n"); return ret; + } } return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, diff --git a/drivers/clk/meson/meson-aoclk.h b/drivers/clk/meson/meson-aoclk.h index ab2819e889228..999cde3868f7f 100644 --- a/drivers/clk/meson/meson-aoclk.h +++ b/drivers/clk/meson/meson-aoclk.h @@ -11,16 +11,27 @@ #ifndef __MESON_AOCLK_H__ #define __MESON_AOCLK_H__ +#include <linux/clk-provider.h> #include <linux/platform_device.h> +#include <linux/regmap.h> #include <linux/reset-controller.h> + #include "clk-regmap.h" +struct meson_aoclk_input { + const char *name; + bool required; +}; + struct meson_aoclk_data { const unsigned int reset_reg; const int num_reset; const unsigned int *reset; - int num_clks; + const int num_clks; struct clk_regmap **clks; + const int num_inputs; + const struct meson_aoclk_input *inputs; + const char *input_prefix; const struct clk_hw_onecell_data *hw_data; }; diff --git a/drivers/clk/meson/meson-eeclk.c b/drivers/clk/meson/meson-eeclk.c new file mode 100644 index 0000000000000..37a34c9c3885e --- /dev/null +++ b/drivers/clk/meson/meson-eeclk.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2019 BayLibre, SAS. + * Author: Jerome Brunet <jbrunet@baylibre.com> + */ + +#include <linux/clk-provider.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> + +#include "clk-input.h" +#include "clk-regmap.h" +#include "meson-eeclk.h" + +int meson_eeclkc_probe(struct platform_device *pdev) +{ + const struct meson_eeclkc_data *data; + struct device *dev = &pdev->dev; + struct clk_hw *input; + struct regmap *map; + int ret, i; + + data = of_device_get_match_data(dev); + if (!data) + return -EINVAL; + + /* Get the hhi system controller node */ + map = syscon_node_to_regmap(of_get_parent(dev->of_node)); + if (IS_ERR(map)) { + dev_err(dev, + "failed to get HHI regmap\n"); + return PTR_ERR(map); + } + + input = meson_clk_hw_register_input(dev, "xtal", IN_PREFIX "xtal", 0); + if (IS_ERR(input)) { + ret = PTR_ERR(input); + if (ret != -EPROBE_DEFER) + dev_err(dev, "failed to get input clock"); + return ret; + } + + /* Populate regmap for the regmap backed clocks */ + for (i = 0; i < data->regmap_clk_num; i++) + data->regmap_clks[i]->map = map; + + for (i = 0; i < data->hw_onecell_data->num; i++) { + /* array might be sparse */ + if (!data->hw_onecell_data->hws[i]) + continue; + + ret = devm_clk_hw_register(dev, data->hw_onecell_data->hws[i]); + if (ret) { + dev_err(dev, "Clock registration failed\n"); + return ret; + } + } + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, + data->hw_onecell_data); +} diff --git a/drivers/clk/meson/meson-eeclk.h b/drivers/clk/meson/meson-eeclk.h new file mode 100644 index 0000000000000..1b809b1419fe0 --- /dev/null +++ b/drivers/clk/meson/meson-eeclk.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019 BayLibre, SAS. + * Author: Jerome Brunet <jbrunet@baylibre.com> + */ + +#ifndef __MESON_CLKC_H +#define __MESON_CLKC_H + +#include <linux/clk-provider.h> +#include "clk-regmap.h" + +#define IN_PREFIX "ee-in-" + +struct platform_device; + +struct meson_eeclkc_data { + struct clk_regmap *const *regmap_clks; + unsigned int regmap_clk_num; + struct clk_hw_onecell_data *hw_onecell_data; +}; + +int meson_eeclkc_probe(struct platform_device *pdev); + +#endif /* __MESON_CLKC_H */ diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c index 950d0e548c75e..576ad42252d04 100644 --- a/drivers/clk/meson/meson8b.c +++ b/drivers/clk/meson/meson8b.c @@ -16,9 +16,10 @@ #include <linux/slab.h> #include <linux/regmap.h> -#include "clkc.h" #include "meson8b.h" #include "clk-regmap.h" +#include "clk-pll.h" +#include "clk-mpll.h" static DEFINE_SPINLOCK(meson_clk_lock); @@ -803,16 +804,16 @@ static struct clk_fixed_factor meson8b_cpu_clk_div8 = { }, }; -static u32 mux_table_abp[] = { 1, 2, 3, 4, 5, 6, 7 }; -static struct clk_regmap meson8b_abp_clk_sel = { +static u32 mux_table_apb[] = { 1, 2, 3, 4, 5, 6, 7 }; +static struct clk_regmap meson8b_apb_clk_sel = { .data = &(struct clk_regmap_mux_data){ .offset = HHI_SYS_CPU_CLK_CNTL1, .mask = 0x7, .shift = 3, - .table = mux_table_abp, + .table = mux_table_apb, }, .hw.init = &(struct clk_init_data){ - .name = "abp_clk_sel", + .name = "apb_clk_sel", .ops = &clk_regmap_mux_ops, .parent_names = (const char *[]){ "cpu_clk_div2", "cpu_clk_div3", @@ -825,16 +826,16 @@ static struct clk_regmap meson8b_abp_clk_sel = { }, }; -static struct clk_regmap meson8b_abp_clk_gate = { +static struct clk_regmap meson8b_apb_clk_gate = { .data = &(struct clk_regmap_gate_data){ .offset = HHI_SYS_CPU_CLK_CNTL1, .bit_idx = 16, .flags = CLK_GATE_SET_TO_DISABLE, }, .hw.init = &(struct clk_init_data){ - .name = "abp_clk_dis", + .name = "apb_clk_dis", .ops = &clk_regmap_gate_ro_ops, - .parent_names = (const char *[]){ "abp_clk_sel" }, + .parent_names = (const char *[]){ "apb_clk_sel" }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, }, @@ -1573,6 +1574,135 @@ static struct clk_regmap meson8b_hdmi_sys = { }, }; +/* + * The MALI IP is clocked by two identical clocks (mali_0 and mali_1) + * muxed by a glitch-free switch on Meson8b and Meson8m2. Meson8 only + * has mali_0 and no glitch-free mux. + */ +static const char * const meson8b_mali_0_1_parent_names[] = { + "xtal", "mpll2", "mpll1", "fclk_div7", "fclk_div4", "fclk_div3", + "fclk_div5" +}; + +static u32 meson8b_mali_0_1_mux_table[] = { 0, 2, 3, 4, 5, 6, 7 }; + +static struct clk_regmap meson8b_mali_0_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_MALI_CLK_CNTL, + .mask = 0x7, + .shift = 9, + .table = meson8b_mali_0_1_mux_table, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_0_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = meson8b_mali_0_1_parent_names, + .num_parents = ARRAY_SIZE(meson8b_mali_0_1_parent_names), + /* + * Don't propagate rate changes up because the only changeable + * parents are mpll1 and mpll2 but we need those for audio and + * RGMII (Ethernet). We don't want to change the audio or + * Ethernet clocks when setting the GPU frequency. + */ + .flags = 0, + }, +}; + +static struct clk_regmap meson8b_mali_0_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_MALI_CLK_CNTL, + .shift = 0, + .width = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_0_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "mali_0_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap meson8b_mali_0 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_MALI_CLK_CNTL, + .bit_idx = 8, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_0", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "mali_0_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap meson8b_mali_1_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_MALI_CLK_CNTL, + .mask = 0x7, + .shift = 25, + .table = meson8b_mali_0_1_mux_table, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_1_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = meson8b_mali_0_1_parent_names, + .num_parents = ARRAY_SIZE(meson8b_mali_0_1_parent_names), + /* + * Don't propagate rate changes up because the only changeable + * parents are mpll1 and mpll2 but we need those for audio and + * RGMII (Ethernet). We don't want to change the audio or + * Ethernet clocks when setting the GPU frequency. + */ + .flags = 0, + }, +}; + +static struct clk_regmap meson8b_mali_1_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_MALI_CLK_CNTL, + .shift = 16, + .width = 7, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_1_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "mali_1_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap meson8b_mali_1 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_MALI_CLK_CNTL, + .bit_idx = 24, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali_1", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "mali_1_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap meson8b_mali = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_MALI_CLK_CNTL, + .mask = 1, + .shift = 31, + }, + .hw.init = &(struct clk_init_data){ + .name = "mali", + .ops = &clk_regmap_mux_ops, + .parent_names = (const char *[]){ "mali_0", "mali_1" }, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + }, +}; + /* Everything Else (EE) domain gates */ static MESON_GATE(meson8b_ddr, HHI_GCLK_MPEG0, 0); @@ -1659,6 +1789,188 @@ static MESON_GATE(meson8b_ao_ahb_sram, HHI_GCLK_AO, 1); static MESON_GATE(meson8b_ao_ahb_bus, HHI_GCLK_AO, 2); static MESON_GATE(meson8b_ao_iface, HHI_GCLK_AO, 3); +static struct clk_hw_onecell_data meson8_hw_onecell_data = { + .hws = { + [CLKID_XTAL] = &meson8b_xtal.hw, + [CLKID_PLL_FIXED] = &meson8b_fixed_pll.hw, + [CLKID_PLL_VID] = &meson8b_vid_pll.hw, + [CLKID_PLL_SYS] = &meson8b_sys_pll.hw, + [CLKID_FCLK_DIV2] = &meson8b_fclk_div2.hw, + [CLKID_FCLK_DIV3] = &meson8b_fclk_div3.hw, + [CLKID_FCLK_DIV4] = &meson8b_fclk_div4.hw, + [CLKID_FCLK_DIV5] = &meson8b_fclk_div5.hw, + [CLKID_FCLK_DIV7] = &meson8b_fclk_div7.hw, + [CLKID_CPUCLK] = &meson8b_cpu_clk.hw, + [CLKID_MPEG_SEL] = &meson8b_mpeg_clk_sel.hw, + [CLKID_MPEG_DIV] = &meson8b_mpeg_clk_div.hw, + [CLKID_CLK81] = &meson8b_clk81.hw, + [CLKID_DDR] = &meson8b_ddr.hw, + [CLKID_DOS] = &meson8b_dos.hw, + [CLKID_ISA] = &meson8b_isa.hw, + [CLKID_PL301] = &meson8b_pl301.hw, + [CLKID_PERIPHS] = &meson8b_periphs.hw, + [CLKID_SPICC] = &meson8b_spicc.hw, + [CLKID_I2C] = &meson8b_i2c.hw, + [CLKID_SAR_ADC] = &meson8b_sar_adc.hw, + [CLKID_SMART_CARD] = &meson8b_smart_card.hw, + [CLKID_RNG0] = &meson8b_rng0.hw, + [CLKID_UART0] = &meson8b_uart0.hw, + [CLKID_SDHC] = &meson8b_sdhc.hw, + [CLKID_STREAM] = &meson8b_stream.hw, + [CLKID_ASYNC_FIFO] = &meson8b_async_fifo.hw, + [CLKID_SDIO] = &meson8b_sdio.hw, + [CLKID_ABUF] = &meson8b_abuf.hw, + [CLKID_HIU_IFACE] = &meson8b_hiu_iface.hw, + [CLKID_ASSIST_MISC] = &meson8b_assist_misc.hw, + [CLKID_SPI] = &meson8b_spi.hw, + [CLKID_I2S_SPDIF] = &meson8b_i2s_spdif.hw, + [CLKID_ETH] = &meson8b_eth.hw, + [CLKID_DEMUX] = &meson8b_demux.hw, + [CLKID_AIU_GLUE] = &meson8b_aiu_glue.hw, + [CLKID_IEC958] = &meson8b_iec958.hw, + [CLKID_I2S_OUT] = &meson8b_i2s_out.hw, + [CLKID_AMCLK] = &meson8b_amclk.hw, + [CLKID_AIFIFO2] = &meson8b_aififo2.hw, + [CLKID_MIXER] = &meson8b_mixer.hw, + [CLKID_MIXER_IFACE] = &meson8b_mixer_iface.hw, + [CLKID_ADC] = &meson8b_adc.hw, + [CLKID_BLKMV] = &meson8b_blkmv.hw, + [CLKID_AIU] = &meson8b_aiu.hw, + [CLKID_UART1] = &meson8b_uart1.hw, + [CLKID_G2D] = &meson8b_g2d.hw, + [CLKID_USB0] = &meson8b_usb0.hw, + [CLKID_USB1] = &meson8b_usb1.hw, + [CLKID_RESET] = &meson8b_reset.hw, + [CLKID_NAND] = &meson8b_nand.hw, + [CLKID_DOS_PARSER] = &meson8b_dos_parser.hw, + [CLKID_USB] = &meson8b_usb.hw, + [CLKID_VDIN1] = &meson8b_vdin1.hw, + [CLKID_AHB_ARB0] = &meson8b_ahb_arb0.hw, + [CLKID_EFUSE] = &meson8b_efuse.hw, + [CLKID_BOOT_ROM] = &meson8b_boot_rom.hw, + [CLKID_AHB_DATA_BUS] = &meson8b_ahb_data_bus.hw, + [CLKID_AHB_CTRL_BUS] = &meson8b_ahb_ctrl_bus.hw, + [CLKID_HDMI_INTR_SYNC] = &meson8b_hdmi_intr_sync.hw, + [CLKID_HDMI_PCLK] = &meson8b_hdmi_pclk.hw, + [CLKID_USB1_DDR_BRIDGE] = &meson8b_usb1_ddr_bridge.hw, + [CLKID_USB0_DDR_BRIDGE] = &meson8b_usb0_ddr_bridge.hw, + [CLKID_MMC_PCLK] = &meson8b_mmc_pclk.hw, + [CLKID_DVIN] = &meson8b_dvin.hw, + [CLKID_UART2] = &meson8b_uart2.hw, + [CLKID_SANA] = &meson8b_sana.hw, + [CLKID_VPU_INTR] = &meson8b_vpu_intr.hw, + [CLKID_SEC_AHB_AHB3_BRIDGE] = &meson8b_sec_ahb_ahb3_bridge.hw, + [CLKID_CLK81_A9] = &meson8b_clk81_a9.hw, + [CLKID_VCLK2_VENCI0] = &meson8b_vclk2_venci0.hw, + [CLKID_VCLK2_VENCI1] = &meson8b_vclk2_venci1.hw, + [CLKID_VCLK2_VENCP0] = &meson8b_vclk2_vencp0.hw, + [CLKID_VCLK2_VENCP1] = &meson8b_vclk2_vencp1.hw, + [CLKID_GCLK_VENCI_INT] = &meson8b_gclk_venci_int.hw, + [CLKID_GCLK_VENCP_INT] = &meson8b_gclk_vencp_int.hw, + [CLKID_DAC_CLK] = &meson8b_dac_clk.hw, + [CLKID_AOCLK_GATE] = &meson8b_aoclk_gate.hw, + [CLKID_IEC958_GATE] = &meson8b_iec958_gate.hw, + [CLKID_ENC480P] = &meson8b_enc480p.hw, + [CLKID_RNG1] = &meson8b_rng1.hw, + [CLKID_GCLK_VENCL_INT] = &meson8b_gclk_vencl_int.hw, + [CLKID_VCLK2_VENCLMCC] = &meson8b_vclk2_venclmcc.hw, + [CLKID_VCLK2_VENCL] = &meson8b_vclk2_vencl.hw, + [CLKID_VCLK2_OTHER] = &meson8b_vclk2_other.hw, + [CLKID_EDP] = &meson8b_edp.hw, + [CLKID_AO_MEDIA_CPU] = &meson8b_ao_media_cpu.hw, + [CLKID_AO_AHB_SRAM] = &meson8b_ao_ahb_sram.hw, + [CLKID_AO_AHB_BUS] = &meson8b_ao_ahb_bus.hw, + [CLKID_AO_IFACE] = &meson8b_ao_iface.hw, + [CLKID_MPLL0] = &meson8b_mpll0.hw, + [CLKID_MPLL1] = &meson8b_mpll1.hw, + [CLKID_MPLL2] = &meson8b_mpll2.hw, + [CLKID_MPLL0_DIV] = &meson8b_mpll0_div.hw, + [CLKID_MPLL1_DIV] = &meson8b_mpll1_div.hw, + [CLKID_MPLL2_DIV] = &meson8b_mpll2_div.hw, + [CLKID_CPU_IN_SEL] = &meson8b_cpu_in_sel.hw, + [CLKID_CPU_IN_DIV2] = &meson8b_cpu_in_div2.hw, + [CLKID_CPU_IN_DIV3] = &meson8b_cpu_in_div3.hw, + [CLKID_CPU_SCALE_DIV] = &meson8b_cpu_scale_div.hw, + [CLKID_CPU_SCALE_OUT_SEL] = &meson8b_cpu_scale_out_sel.hw, + [CLKID_MPLL_PREDIV] = &meson8b_mpll_prediv.hw, + [CLKID_FCLK_DIV2_DIV] = &meson8b_fclk_div2_div.hw, + [CLKID_FCLK_DIV3_DIV] = &meson8b_fclk_div3_div.hw, + [CLKID_FCLK_DIV4_DIV] = &meson8b_fclk_div4_div.hw, + [CLKID_FCLK_DIV5_DIV] = &meson8b_fclk_div5_div.hw, + [CLKID_FCLK_DIV7_DIV] = &meson8b_fclk_div7_div.hw, + [CLKID_NAND_SEL] = &meson8b_nand_clk_sel.hw, + [CLKID_NAND_DIV] = &meson8b_nand_clk_div.hw, + [CLKID_NAND_CLK] = &meson8b_nand_clk_gate.hw, + [CLKID_PLL_FIXED_DCO] = &meson8b_fixed_pll_dco.hw, + [CLKID_HDMI_PLL_DCO] = &meson8b_hdmi_pll_dco.hw, + [CLKID_PLL_SYS_DCO] = &meson8b_sys_pll_dco.hw, + [CLKID_CPU_CLK_DIV2] = &meson8b_cpu_clk_div2.hw, + [CLKID_CPU_CLK_DIV3] = &meson8b_cpu_clk_div3.hw, + [CLKID_CPU_CLK_DIV4] = &meson8b_cpu_clk_div4.hw, + [CLKID_CPU_CLK_DIV5] = &meson8b_cpu_clk_div5.hw, + [CLKID_CPU_CLK_DIV6] = &meson8b_cpu_clk_div6.hw, + [CLKID_CPU_CLK_DIV7] = &meson8b_cpu_clk_div7.hw, + [CLKID_CPU_CLK_DIV8] = &meson8b_cpu_clk_div8.hw, + [CLKID_APB_SEL] = &meson8b_apb_clk_sel.hw, + [CLKID_APB] = &meson8b_apb_clk_gate.hw, + [CLKID_PERIPH_SEL] = &meson8b_periph_clk_sel.hw, + [CLKID_PERIPH] = &meson8b_periph_clk_gate.hw, + [CLKID_AXI_SEL] = &meson8b_axi_clk_sel.hw, + [CLKID_AXI] = &meson8b_axi_clk_gate.hw, + [CLKID_L2_DRAM_SEL] = &meson8b_l2_dram_clk_sel.hw, + [CLKID_L2_DRAM] = &meson8b_l2_dram_clk_gate.hw, + [CLKID_HDMI_PLL_LVDS_OUT] = &meson8b_hdmi_pll_lvds_out.hw, + [CLKID_HDMI_PLL_HDMI_OUT] = &meson8b_hdmi_pll_hdmi_out.hw, + [CLKID_VID_PLL_IN_SEL] = &meson8b_vid_pll_in_sel.hw, + [CLKID_VID_PLL_IN_EN] = &meson8b_vid_pll_in_en.hw, + [CLKID_VID_PLL_PRE_DIV] = &meson8b_vid_pll_pre_div.hw, + [CLKID_VID_PLL_POST_DIV] = &meson8b_vid_pll_post_div.hw, + [CLKID_VID_PLL_FINAL_DIV] = &meson8b_vid_pll_final_div.hw, + [CLKID_VCLK_IN_SEL] = &meson8b_vclk_in_sel.hw, + [CLKID_VCLK_IN_EN] = &meson8b_vclk_in_en.hw, + [CLKID_VCLK_DIV1] = &meson8b_vclk_div1_gate.hw, + [CLKID_VCLK_DIV2_DIV] = &meson8b_vclk_div2_div.hw, + [CLKID_VCLK_DIV2] = &meson8b_vclk_div2_div_gate.hw, + [CLKID_VCLK_DIV4_DIV] = &meson8b_vclk_div4_div.hw, + [CLKID_VCLK_DIV4] = &meson8b_vclk_div4_div_gate.hw, + [CLKID_VCLK_DIV6_DIV] = &meson8b_vclk_div6_div.hw, + [CLKID_VCLK_DIV6] = &meson8b_vclk_div6_div_gate.hw, + [CLKID_VCLK_DIV12_DIV] = &meson8b_vclk_div12_div.hw, + [CLKID_VCLK_DIV12] = &meson8b_vclk_div12_div_gate.hw, + [CLKID_VCLK2_IN_SEL] = &meson8b_vclk2_in_sel.hw, + [CLKID_VCLK2_IN_EN] = &meson8b_vclk2_clk_in_en.hw, + [CLKID_VCLK2_DIV1] = &meson8b_vclk2_div1_gate.hw, + [CLKID_VCLK2_DIV2_DIV] = &meson8b_vclk2_div2_div.hw, + [CLKID_VCLK2_DIV2] = &meson8b_vclk2_div2_div_gate.hw, + [CLKID_VCLK2_DIV4_DIV] = &meson8b_vclk2_div4_div.hw, + [CLKID_VCLK2_DIV4] = &meson8b_vclk2_div4_div_gate.hw, + [CLKID_VCLK2_DIV6_DIV] = &meson8b_vclk2_div6_div.hw, + [CLKID_VCLK2_DIV6] = &meson8b_vclk2_div6_div_gate.hw, + [CLKID_VCLK2_DIV12_DIV] = &meson8b_vclk2_div12_div.hw, + [CLKID_VCLK2_DIV12] = &meson8b_vclk2_div12_div_gate.hw, + [CLKID_CTS_ENCT_SEL] = &meson8b_cts_enct_sel.hw, + [CLKID_CTS_ENCT] = &meson8b_cts_enct.hw, + [CLKID_CTS_ENCP_SEL] = &meson8b_cts_encp_sel.hw, + [CLKID_CTS_ENCP] = &meson8b_cts_encp.hw, + [CLKID_CTS_ENCI_SEL] = &meson8b_cts_enci_sel.hw, + [CLKID_CTS_ENCI] = &meson8b_cts_enci.hw, + [CLKID_HDMI_TX_PIXEL_SEL] = &meson8b_hdmi_tx_pixel_sel.hw, + [CLKID_HDMI_TX_PIXEL] = &meson8b_hdmi_tx_pixel.hw, + [CLKID_CTS_ENCL_SEL] = &meson8b_cts_encl_sel.hw, + [CLKID_CTS_ENCL] = &meson8b_cts_encl.hw, + [CLKID_CTS_VDAC0_SEL] = &meson8b_cts_vdac0_sel.hw, + [CLKID_CTS_VDAC0] = &meson8b_cts_vdac0.hw, + [CLKID_HDMI_SYS_SEL] = &meson8b_hdmi_sys_sel.hw, + [CLKID_HDMI_SYS_DIV] = &meson8b_hdmi_sys_div.hw, + [CLKID_HDMI_SYS] = &meson8b_hdmi_sys.hw, + [CLKID_MALI_0_SEL] = &meson8b_mali_0_sel.hw, + [CLKID_MALI_0_DIV] = &meson8b_mali_0_div.hw, + [CLKID_MALI] = &meson8b_mali_0.hw, + [CLK_NR_CLKS] = NULL, + }, + .num = CLK_NR_CLKS, +}; + static struct clk_hw_onecell_data meson8b_hw_onecell_data = { .hws = { [CLKID_XTAL] = &meson8b_xtal.hw, @@ -1781,8 +2093,8 @@ static struct clk_hw_onecell_data meson8b_hw_onecell_data = { [CLKID_CPU_CLK_DIV6] = &meson8b_cpu_clk_div6.hw, [CLKID_CPU_CLK_DIV7] = &meson8b_cpu_clk_div7.hw, [CLKID_CPU_CLK_DIV8] = &meson8b_cpu_clk_div8.hw, - [CLKID_ABP_SEL] = &meson8b_abp_clk_sel.hw, - [CLKID_ABP] = &meson8b_abp_clk_gate.hw, + [CLKID_APB_SEL] = &meson8b_apb_clk_sel.hw, + [CLKID_APB] = &meson8b_apb_clk_gate.hw, [CLKID_PERIPH_SEL] = &meson8b_periph_clk_sel.hw, [CLKID_PERIPH] = &meson8b_periph_clk_gate.hw, [CLKID_AXI_SEL] = &meson8b_axi_clk_sel.hw, @@ -1833,6 +2145,13 @@ static struct clk_hw_onecell_data meson8b_hw_onecell_data = { [CLKID_HDMI_SYS_SEL] = &meson8b_hdmi_sys_sel.hw, [CLKID_HDMI_SYS_DIV] = &meson8b_hdmi_sys_div.hw, [CLKID_HDMI_SYS] = &meson8b_hdmi_sys.hw, + [CLKID_MALI_0_SEL] = &meson8b_mali_0_sel.hw, + [CLKID_MALI_0_DIV] = &meson8b_mali_0_div.hw, + [CLKID_MALI_0] = &meson8b_mali_0.hw, + [CLKID_MALI_1_SEL] = &meson8b_mali_1_sel.hw, + [CLKID_MALI_1_DIV] = &meson8b_mali_1_div.hw, + [CLKID_MALI_1] = &meson8b_mali_1.hw, + [CLKID_MALI] = &meson8b_mali.hw, [CLK_NR_CLKS] = NULL, }, .num = CLK_NR_CLKS, @@ -1943,8 +2262,8 @@ static struct clk_regmap *const meson8b_clk_regmaps[] = { &meson8b_fixed_pll_dco, &meson8b_hdmi_pll_dco, &meson8b_sys_pll_dco, - &meson8b_abp_clk_sel, - &meson8b_abp_clk_gate, + &meson8b_apb_clk_sel, + &meson8b_apb_clk_gate, &meson8b_periph_clk_sel, &meson8b_periph_clk_gate, &meson8b_axi_clk_sel, @@ -1988,6 +2307,13 @@ static struct clk_regmap *const meson8b_clk_regmaps[] = { &meson8b_hdmi_sys_sel, &meson8b_hdmi_sys_div, &meson8b_hdmi_sys, + &meson8b_mali_0_sel, + &meson8b_mali_0_div, + &meson8b_mali_0, + &meson8b_mali_1_sel, + &meson8b_mali_1_div, + &meson8b_mali_1, + &meson8b_mali, }; static const struct meson8b_clk_reset_line { @@ -2132,7 +2458,6 @@ static int meson8b_cpu_clk_notifier_cb(struct notifier_block *nb, static struct meson8b_nb_data meson8b_cpu_nb_data = { .nb.notifier_call = meson8b_cpu_clk_notifier_cb, - .onecell_data = &meson8b_hw_onecell_data, }; static const struct regmap_config clkc_regmap_config = { @@ -2141,7 +2466,8 @@ static const struct regmap_config clkc_regmap_config = { .reg_stride = 4, }; -static void __init meson8b_clkc_init(struct device_node *np) +static void __init meson8b_clkc_init_common(struct device_node *np, + struct clk_hw_onecell_data *clk_hw_onecell_data) { struct meson8b_clk_reset *rstc; const char *notifier_clk_name; @@ -2192,14 +2518,16 @@ static void __init meson8b_clkc_init(struct device_node *np) */ for (i = CLKID_XTAL; i < CLK_NR_CLKS; i++) { /* array might be sparse */ - if (!meson8b_hw_onecell_data.hws[i]) + if (!clk_hw_onecell_data->hws[i]) continue; - ret = clk_hw_register(NULL, meson8b_hw_onecell_data.hws[i]); + ret = clk_hw_register(NULL, clk_hw_onecell_data->hws[i]); if (ret) return; } + meson8b_cpu_nb_data.onecell_data = clk_hw_onecell_data; + /* * FIXME we shouldn't program the muxes in notifier handlers. The * tricky programming sequence will be handled by the forthcoming @@ -2215,13 +2543,23 @@ static void __init meson8b_clkc_init(struct device_node *np) } ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, - &meson8b_hw_onecell_data); + clk_hw_onecell_data); if (ret) pr_err("%s: failed to register clock provider\n", __func__); } +static void __init meson8_clkc_init(struct device_node *np) +{ + return meson8b_clkc_init_common(np, &meson8_hw_onecell_data); +} + +static void __init meson8b_clkc_init(struct device_node *np) +{ + return meson8b_clkc_init_common(np, &meson8b_hw_onecell_data); +} + CLK_OF_DECLARE_DRIVER(meson8_clkc, "amlogic,meson8-clkc", - meson8b_clkc_init); + meson8_clkc_init); CLK_OF_DECLARE_DRIVER(meson8b_clkc, "amlogic,meson8b-clkc", meson8b_clkc_init); CLK_OF_DECLARE_DRIVER(meson8m2_clkc, "amlogic,meson8m2-clkc", diff --git a/drivers/clk/meson/meson8b.h b/drivers/clk/meson/meson8b.h index 87fba739af818..b8c58faeae525 100644 --- a/drivers/clk/meson/meson8b.h +++ b/drivers/clk/meson/meson8b.h @@ -33,6 +33,7 @@ #define HHI_VID_CLK_CNTL2 0x194 /* 0x65 offset in data sheet */ #define HHI_VID_DIVIDER_CNTL 0x198 /* 0x66 offset in data sheet */ #define HHI_SYS_CPU_CLK_CNTL0 0x19c /* 0x67 offset in data sheet */ +#define HHI_MALI_CLK_CNTL 0x1b0 /* 0x6c offset in data sheet */ #define HHI_HDMI_CLK_CNTL 0x1cc /* 0x73 offset in data sheet */ #define HHI_NAND_CLK_CNTL 0x25c /* 0x97 offset in data sheet */ #define HHI_MPLL_CNTL 0x280 /* 0xa0 offset in data sheet */ @@ -91,7 +92,7 @@ #define CLKID_CPU_CLK_DIV6 120 #define CLKID_CPU_CLK_DIV7 121 #define CLKID_CPU_CLK_DIV8 122 -#define CLKID_ABP_SEL 123 +#define CLKID_APB_SEL 123 #define CLKID_PERIPH_SEL 125 #define CLKID_AXI_SEL 127 #define CLKID_L2_DRAM_SEL 129 @@ -139,8 +140,14 @@ #define CLKID_HDMI_SYS_SEL 172 #define CLKID_HDMI_SYS_DIV 173 #define CLKID_HDMI_SYS 174 +#define CLKID_MALI_0_SEL 175 +#define CLKID_MALI_0_DIV 176 +#define CLKID_MALI_0 177 +#define CLKID_MALI_1_SEL 178 +#define CLKID_MALI_1_DIV 179 +#define CLKID_MALI_1 180 -#define CLK_NR_CLKS 175 +#define CLK_NR_CLKS 181 /* * include the CLKID and RESETID that have diff --git a/drivers/clk/meson/parm.h b/drivers/clk/meson/parm.h new file mode 100644 index 0000000000000..3c9ef1b505ced --- /dev/null +++ b/drivers/clk/meson/parm.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2015 Endless Mobile, Inc. + * Author: Carlo Caione <carlo@endlessm.com> + */ + +#ifndef __MESON_PARM_H +#define __MESON_PARM_H + +#include <linux/bits.h> +#include <linux/regmap.h> + +#define PMASK(width) GENMASK(width - 1, 0) +#define SETPMASK(width, shift) GENMASK(shift + width - 1, shift) +#define CLRPMASK(width, shift) (~SETPMASK(width, shift)) + +#define PARM_GET(width, shift, reg) \ + (((reg) & SETPMASK(width, shift)) >> (shift)) +#define PARM_SET(width, shift, reg, val) \ + (((reg) & CLRPMASK(width, shift)) | ((val) << (shift))) + +#define MESON_PARM_APPLICABLE(p) (!!((p)->width)) + +struct parm { + u16 reg_off; + u8 shift; + u8 width; +}; + +static inline unsigned int meson_parm_read(struct regmap *map, struct parm *p) +{ + unsigned int val; + + regmap_read(map, p->reg_off, &val); + return PARM_GET(p->width, p->shift, val); +} + +static inline void meson_parm_write(struct regmap *map, struct parm *p, + unsigned int val) +{ + regmap_update_bits(map, p->reg_off, SETPMASK(p->width, p->shift), + val << p->shift); +} + +#endif /* __MESON_PARM_H */ + diff --git a/drivers/clk/meson/sclk-div.c b/drivers/clk/meson/sclk-div.c index bc64019b8eeb1..3acf037802215 100644 --- a/drivers/clk/meson/sclk-div.c +++ b/drivers/clk/meson/sclk-div.c @@ -16,7 +16,11 @@ * duty_cycle = (1 + hi) / (1 + val) */ -#include "clkc-audio.h" +#include <linux/clk-provider.h> +#include <linux/module.h> + +#include "clk-regmap.h" +#include "sclk-div.h" static inline struct meson_sclk_div_data * meson_sclk_div_data(struct clk_regmap *clk) @@ -241,3 +245,7 @@ const struct clk_ops meson_sclk_div_ops = { .init = sclk_div_init, }; EXPORT_SYMBOL_GPL(meson_sclk_div_ops); + +MODULE_DESCRIPTION("Amlogic Sample divider driver"); +MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/meson/clkc-audio.h b/drivers/clk/meson/sclk-div.h index 0a7c157ebf816..b64b2a32005f0 100644 --- a/drivers/clk/meson/clkc-audio.h +++ b/drivers/clk/meson/sclk-div.h @@ -4,16 +4,11 @@ * Author: Jerome Brunet <jbrunet@baylibre.com> */ -#ifndef __MESON_CLKC_AUDIO_H -#define __MESON_CLKC_AUDIO_H +#ifndef __MESON_SCLK_DIV_H +#define __MESON_SCLK_DIV_H -#include "clkc.h" - -struct meson_clk_triphase_data { - struct parm ph0; - struct parm ph1; - struct parm ph2; -}; +#include <linux/clk-provider.h> +#include "parm.h" struct meson_sclk_div_data { struct parm div; @@ -22,7 +17,6 @@ struct meson_sclk_div_data { struct clk_duty cached_duty; }; -extern const struct clk_ops meson_clk_triphase_ops; extern const struct clk_ops meson_sclk_div_ops; -#endif /* __MESON_CLKC_AUDIO_H */ +#endif /* __MESON_SCLK_DIV_H */ diff --git a/drivers/clk/meson/vid-pll-div.c b/drivers/clk/meson/vid-pll-div.c index 88af0e282ea06..08bcc01c09238 100644 --- a/drivers/clk/meson/vid-pll-div.c +++ b/drivers/clk/meson/vid-pll-div.c @@ -5,7 +5,10 @@ */ #include <linux/clk-provider.h> -#include "clkc.h" +#include <linux/module.h> + +#include "clk-regmap.h" +#include "vid-pll-div.h" static inline struct meson_vid_pll_div_data * meson_vid_pll_div_data(struct clk_regmap *clk) @@ -89,3 +92,8 @@ static unsigned long meson_vid_pll_div_recalc_rate(struct clk_hw *hw, const struct clk_ops meson_vid_pll_div_ro_ops = { .recalc_rate = meson_vid_pll_div_recalc_rate, }; +EXPORT_SYMBOL_GPL(meson_vid_pll_div_ro_ops); + +MODULE_DESCRIPTION("Amlogic video pll divider driver"); +MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/meson/vid-pll-div.h b/drivers/clk/meson/vid-pll-div.h new file mode 100644 index 0000000000000..c0128e33ccf90 --- /dev/null +++ b/drivers/clk/meson/vid-pll-div.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019 BayLibre, SAS. + * Author: Jerome Brunet <jbrunet@baylibre.com> + */ + +#ifndef __MESON_VID_PLL_DIV_H +#define __MESON_VID_PLL_DIV_H + +#include <linux/clk-provider.h> +#include "parm.h" + +struct meson_vid_pll_div_data { + struct parm val; + struct parm sel; +}; + +extern const struct clk_ops meson_vid_pll_div_ro_ops; + +#endif /* __MESON_VID_PLL_DIV_H */ diff --git a/drivers/clk/mmp/clk-of-mmp2.c b/drivers/clk/mmp/clk-of-mmp2.c index d083b860f0833..a60a1be937ad6 100644 --- a/drivers/clk/mmp/clk-of-mmp2.c +++ b/drivers/clk/mmp/clk-of-mmp2.c @@ -229,9 +229,10 @@ static struct mmp_param_gate_clk apmu_gate_clks[] = { {MMP2_CLK_SDH1, "sdh1_clk", "sdh_mix_clk", CLK_SET_RATE_PARENT, APMU_SDH1, 0x1b, 0x1b, 0x0, 0, &sdh_lock}, {MMP2_CLK_SDH2, "sdh2_clk", "sdh_mix_clk", CLK_SET_RATE_PARENT, APMU_SDH2, 0x1b, 0x1b, 0x0, 0, &sdh_lock}, {MMP2_CLK_SDH3, "sdh3_clk", "sdh_mix_clk", CLK_SET_RATE_PARENT, APMU_SDH3, 0x1b, 0x1b, 0x0, 0, &sdh_lock}, - {MMP2_CLK_DISP0, "disp0_clk", "disp0_div", CLK_SET_RATE_PARENT, APMU_DISP0, 0x1b, 0x1b, 0x0, 0, &disp0_lock}, + {MMP2_CLK_DISP0, "disp0_clk", "disp0_div", CLK_SET_RATE_PARENT, APMU_DISP0, 0x09, 0x09, 0x0, 0, &disp0_lock}, + {MMP2_CLK_DISP0_LCDC, "disp0_lcdc_clk", "disp0_mux", CLK_SET_RATE_PARENT, APMU_DISP0, 0x12, 0x12, 0x0, 0, &disp0_lock}, {MMP2_CLK_DISP0_SPHY, "disp0_sphy_clk", "disp0_sphy_div", CLK_SET_RATE_PARENT, APMU_DISP0, 0x1024, 0x1024, 0x0, 0, &disp0_lock}, - {MMP2_CLK_DISP1, "disp1_clk", "disp1_div", CLK_SET_RATE_PARENT, APMU_DISP1, 0x1b, 0x1b, 0x0, 0, &disp1_lock}, + {MMP2_CLK_DISP1, "disp1_clk", "disp1_div", CLK_SET_RATE_PARENT, APMU_DISP1, 0x09, 0x09, 0x0, 0, &disp1_lock}, {MMP2_CLK_CCIC_ARBITER, "ccic_arbiter", "vctcxo", CLK_SET_RATE_PARENT, APMU_CCIC0, 0x1800, 0x1800, 0x0, 0, &ccic0_lock}, {MMP2_CLK_CCIC0, "ccic0_clk", "ccic0_mix_clk", CLK_SET_RATE_PARENT, APMU_CCIC0, 0x1b, 0x1b, 0x0, 0, &ccic0_lock}, {MMP2_CLK_CCIC0_PHY, "ccic0_phy_clk", "ccic0_mix_clk", CLK_SET_RATE_PARENT, APMU_CCIC0, 0x24, 0x24, 0x0, 0, &ccic0_lock}, diff --git a/drivers/clk/mvebu/armada-370.c b/drivers/clk/mvebu/armada-370.c index 7dedfaa6e1520..5c6bbee396b3c 100644 --- a/drivers/clk/mvebu/armada-370.c +++ b/drivers/clk/mvebu/armada-370.c @@ -175,8 +175,10 @@ static void __init a370_clk_init(struct device_node *np) mvebu_coreclk_setup(np, &a370_coreclks); - if (cgnp) + if (cgnp) { mvebu_clk_gating_setup(cgnp, a370_gating_desc); + of_node_put(cgnp); + } } CLK_OF_DECLARE(a370_clk, "marvell,armada-370-core-clock", a370_clk_init); diff --git a/drivers/clk/mvebu/armada-xp.c b/drivers/clk/mvebu/armada-xp.c index e8f03293ec833..fa1568279c236 100644 --- a/drivers/clk/mvebu/armada-xp.c +++ b/drivers/clk/mvebu/armada-xp.c @@ -226,7 +226,9 @@ static void __init axp_clk_init(struct device_node *np) mvebu_coreclk_setup(np, &axp_coreclks); - if (cgnp) + if (cgnp) { mvebu_clk_gating_setup(cgnp, axp_gating_desc); + of_node_put(cgnp); + } } CLK_OF_DECLARE(axp_clk, "marvell,armada-xp-core-clock", axp_clk_init); diff --git a/drivers/clk/mvebu/dove.c b/drivers/clk/mvebu/dove.c index e0dd99f36bf43..0bd09d33f9cfe 100644 --- a/drivers/clk/mvebu/dove.c +++ b/drivers/clk/mvebu/dove.c @@ -188,10 +188,14 @@ static void __init dove_clk_init(struct device_node *np) mvebu_coreclk_setup(np, &dove_coreclks); - if (ddnp) + if (ddnp) { dove_divider_clk_init(ddnp); + of_node_put(ddnp); + } - if (cgnp) + if (cgnp) { mvebu_clk_gating_setup(cgnp, dove_gating_desc); + of_node_put(cgnp); + } } CLK_OF_DECLARE(dove_clk, "marvell,dove-core-clock", dove_clk_init); diff --git a/drivers/clk/mvebu/kirkwood.c b/drivers/clk/mvebu/kirkwood.c index 6f784167bda4a..35af3aa18f1c4 100644 --- a/drivers/clk/mvebu/kirkwood.c +++ b/drivers/clk/mvebu/kirkwood.c @@ -331,6 +331,8 @@ static void __init kirkwood_clk_init(struct device_node *np) if (cgnp) { mvebu_clk_gating_setup(cgnp, kirkwood_gating_desc); kirkwood_clk_muxing_setup(cgnp, kirkwood_mux_desc); + + of_node_put(cgnp); } } CLK_OF_DECLARE(kirkwood_clk, "marvell,kirkwood-core-clock", diff --git a/drivers/clk/mvebu/mv98dx3236.c b/drivers/clk/mvebu/mv98dx3236.c index 0a74cf7a7725a..1c8ab4f834bab 100644 --- a/drivers/clk/mvebu/mv98dx3236.c +++ b/drivers/clk/mvebu/mv98dx3236.c @@ -172,7 +172,9 @@ static void __init mv98dx3236_clk_init(struct device_node *np) mvebu_coreclk_setup(np, &mv98dx3236_core_clocks); - if (cgnp) + if (cgnp) { mvebu_clk_gating_setup(cgnp, mv98dx3236_gating_desc); + of_node_put(cgnp); + } } CLK_OF_DECLARE(mv98dx3236_clk, "marvell,mv98dx3236-core-clock", mv98dx3236_clk_init); diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h index e5eca8a1abe48..c25b57c3cbc86 100644 --- a/drivers/clk/qcom/clk-rcg.h +++ b/drivers/clk/qcom/clk-rcg.h @@ -71,7 +71,6 @@ struct src_sel { * @freq_tbl: frequency table * @clkr: regmap clock handle * @lock: register lock - * */ struct clk_rcg { u32 ns_reg; @@ -107,7 +106,6 @@ extern const struct clk_ops clk_rcg_lcc_ops; * @freq_tbl: frequency table * @clkr: regmap clock handle * @lock: register lock - * */ struct clk_dyn_rcg { u32 ns_reg[2]; @@ -140,7 +138,7 @@ extern const struct clk_ops clk_dyn_rcg_ops; * @parent_map: map from software's parent index to hardware's src_sel field * @freq_tbl: frequency table * @clkr: regmap clock handle - * + * @cfg_off: defines the cfg register offset from the CMD_RCGR + CFG_REG */ struct clk_rcg2 { u32 cmd_rcgr; @@ -150,6 +148,7 @@ struct clk_rcg2 { const struct parent_map *parent_map; const struct freq_tbl *freq_tbl; struct clk_regmap clkr; + u8 cfg_off; }; #define to_clk_rcg2(_hw) container_of(to_clk_regmap(_hw), struct clk_rcg2, clkr) diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index 6e3bd195d012f..8c02bffe50dfa 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -41,6 +41,11 @@ #define N_REG 0xc #define D_REG 0x10 +#define RCG_CFG_OFFSET(rcg) ((rcg)->cmd_rcgr + (rcg)->cfg_off + CFG_REG) +#define RCG_M_OFFSET(rcg) ((rcg)->cmd_rcgr + (rcg)->cfg_off + M_REG) +#define RCG_N_OFFSET(rcg) ((rcg)->cmd_rcgr + (rcg)->cfg_off + N_REG) +#define RCG_D_OFFSET(rcg) ((rcg)->cmd_rcgr + (rcg)->cfg_off + D_REG) + /* Dynamic Frequency Scaling */ #define MAX_PERF_LEVEL 8 #define SE_CMD_DFSR_OFFSET 0x14 @@ -74,7 +79,7 @@ static u8 clk_rcg2_get_parent(struct clk_hw *hw) u32 cfg; int i, ret; - ret = regmap_read(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG, &cfg); + ret = regmap_read(rcg->clkr.regmap, RCG_CFG_OFFSET(rcg), &cfg); if (ret) goto err; @@ -123,7 +128,7 @@ static int clk_rcg2_set_parent(struct clk_hw *hw, u8 index) int ret; u32 cfg = rcg->parent_map[index].cfg << CFG_SRC_SEL_SHIFT; - ret = regmap_update_bits(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG, + ret = regmap_update_bits(rcg->clkr.regmap, RCG_CFG_OFFSET(rcg), CFG_SRC_SEL_MASK, cfg); if (ret) return ret; @@ -162,13 +167,13 @@ clk_rcg2_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) struct clk_rcg2 *rcg = to_clk_rcg2(hw); u32 cfg, hid_div, m = 0, n = 0, mode = 0, mask; - regmap_read(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG, &cfg); + regmap_read(rcg->clkr.regmap, RCG_CFG_OFFSET(rcg), &cfg); if (rcg->mnd_width) { mask = BIT(rcg->mnd_width) - 1; - regmap_read(rcg->clkr.regmap, rcg->cmd_rcgr + M_REG, &m); + regmap_read(rcg->clkr.regmap, RCG_M_OFFSET(rcg), &m); m &= mask; - regmap_read(rcg->clkr.regmap, rcg->cmd_rcgr + N_REG, &n); + regmap_read(rcg->clkr.regmap, RCG_N_OFFSET(rcg), &n); n = ~n; n &= mask; n += m; @@ -263,17 +268,17 @@ static int __clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f) if (rcg->mnd_width && f->n) { mask = BIT(rcg->mnd_width) - 1; ret = regmap_update_bits(rcg->clkr.regmap, - rcg->cmd_rcgr + M_REG, mask, f->m); + RCG_M_OFFSET(rcg), mask, f->m); if (ret) return ret; ret = regmap_update_bits(rcg->clkr.regmap, - rcg->cmd_rcgr + N_REG, mask, ~(f->n - f->m)); + RCG_N_OFFSET(rcg), mask, ~(f->n - f->m)); if (ret) return ret; ret = regmap_update_bits(rcg->clkr.regmap, - rcg->cmd_rcgr + D_REG, mask, ~f->n); + RCG_D_OFFSET(rcg), mask, ~f->n); if (ret) return ret; } @@ -284,8 +289,7 @@ static int __clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f) cfg |= rcg->parent_map[index].cfg << CFG_SRC_SEL_SHIFT; if (rcg->mnd_width && f->n && (f->m != f->n)) cfg |= CFG_MODE_DUAL_EDGE; - - return regmap_update_bits(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG, + return regmap_update_bits(rcg->clkr.regmap, RCG_CFG_OFFSET(rcg), mask, cfg); } diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c index 9f4fc7773fb28..c3fd632af1190 100644 --- a/drivers/clk/qcom/clk-rpmh.c +++ b/drivers/clk/qcom/clk-rpmh.c @@ -18,6 +18,31 @@ #define CLK_RPMH_ARC_EN_OFFSET 0 #define CLK_RPMH_VRM_EN_OFFSET 4 +#define BCM_TCS_CMD_COMMIT_MASK 0x40000000 +#define BCM_TCS_CMD_VALID_SHIFT 29 +#define BCM_TCS_CMD_VOTE_MASK 0x3fff +#define BCM_TCS_CMD_VOTE_SHIFT 0 + +#define BCM_TCS_CMD(valid, vote) \ + (BCM_TCS_CMD_COMMIT_MASK | \ + ((valid) << BCM_TCS_CMD_VALID_SHIFT) | \ + ((vote & BCM_TCS_CMD_VOTE_MASK) \ + << BCM_TCS_CMD_VOTE_SHIFT)) + +/** + * struct bcm_db - Auxiliary data pertaining to each Bus Clock Manager(BCM) + * @unit: divisor used to convert Hz value to an RPMh msg + * @width: multiplier used to convert Hz value to an RPMh msg + * @vcd: virtual clock domain that this bcm belongs to + * @reserved: reserved to pad the struct + */ +struct bcm_db { + __le32 unit; + __le16 width; + u8 vcd; + u8 reserved; +}; + /** * struct clk_rpmh - individual rpmh clock data structure * @hw: handle between common and hardware-specific interfaces @@ -29,6 +54,7 @@ * @aggr_state: rpmh clock aggregated state * @last_sent_aggr_state: rpmh clock last aggr state sent to RPMh * @valid_state_mask: mask to determine the state of the rpmh clock + * @unit: divisor to convert rate to rpmh msg in magnitudes of Khz * @dev: device to which it is attached * @peer: pointer to the clock rpmh sibling */ @@ -42,6 +68,7 @@ struct clk_rpmh { u32 aggr_state; u32 last_sent_aggr_state; u32 valid_state_mask; + u32 unit; struct device *dev; struct clk_rpmh *peer; }; @@ -98,6 +125,17 @@ static DEFINE_MUTEX(rpmh_clk_lock); __DEFINE_CLK_RPMH(_platform, _name, _name_active, _res_name, \ CLK_RPMH_VRM_EN_OFFSET, 1, _div) +#define DEFINE_CLK_RPMH_BCM(_platform, _name, _res_name) \ + static struct clk_rpmh _platform##_##_name = { \ + .res_name = _res_name, \ + .valid_state_mask = BIT(RPMH_ACTIVE_ONLY_STATE), \ + .div = 1, \ + .hw.init = &(struct clk_init_data){ \ + .ops = &clk_rpmh_bcm_ops, \ + .name = #_name, \ + }, \ + } + static inline struct clk_rpmh *to_clk_rpmh(struct clk_hw *_hw) { return container_of(_hw, struct clk_rpmh, hw); @@ -210,6 +248,96 @@ static const struct clk_ops clk_rpmh_ops = { .recalc_rate = clk_rpmh_recalc_rate, }; +static int clk_rpmh_bcm_send_cmd(struct clk_rpmh *c, bool enable) +{ + struct tcs_cmd cmd = { 0 }; + u32 cmd_state; + int ret; + + mutex_lock(&rpmh_clk_lock); + + cmd_state = 0; + if (enable) { + cmd_state = 1; + if (c->aggr_state) + cmd_state = c->aggr_state; + } + + if (c->last_sent_aggr_state == cmd_state) { + mutex_unlock(&rpmh_clk_lock); + return 0; + } + + cmd.addr = c->res_addr; + cmd.data = BCM_TCS_CMD(enable, cmd_state); + + ret = rpmh_write_async(c->dev, RPMH_ACTIVE_ONLY_STATE, &cmd, 1); + if (ret) { + dev_err(c->dev, "set active state of %s failed: (%d)\n", + c->res_name, ret); + mutex_unlock(&rpmh_clk_lock); + return ret; + } + + c->last_sent_aggr_state = cmd_state; + + mutex_unlock(&rpmh_clk_lock); + + return 0; +} + +static int clk_rpmh_bcm_prepare(struct clk_hw *hw) +{ + struct clk_rpmh *c = to_clk_rpmh(hw); + + return clk_rpmh_bcm_send_cmd(c, true); +}; + +static void clk_rpmh_bcm_unprepare(struct clk_hw *hw) +{ + struct clk_rpmh *c = to_clk_rpmh(hw); + + clk_rpmh_bcm_send_cmd(c, false); +}; + +static int clk_rpmh_bcm_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_rpmh *c = to_clk_rpmh(hw); + + c->aggr_state = rate / c->unit; + /* + * Since any non-zero value sent to hw would result in enabling the + * clock, only send the value if the clock has already been prepared. + */ + if (clk_hw_is_prepared(hw)) + clk_rpmh_bcm_send_cmd(c, true); + + return 0; +}; + +static long clk_rpmh_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + return rate; +} + +static unsigned long clk_rpmh_bcm_recalc_rate(struct clk_hw *hw, + unsigned long prate) +{ + struct clk_rpmh *c = to_clk_rpmh(hw); + + return c->aggr_state * c->unit; +} + +static const struct clk_ops clk_rpmh_bcm_ops = { + .prepare = clk_rpmh_bcm_prepare, + .unprepare = clk_rpmh_bcm_unprepare, + .set_rate = clk_rpmh_bcm_set_rate, + .round_rate = clk_rpmh_round_rate, + .recalc_rate = clk_rpmh_bcm_recalc_rate, +}; + /* Resource name must match resource id present in cmd-db. */ DEFINE_CLK_RPMH_ARC(sdm845, bi_tcxo, bi_tcxo_ao, "xo.lvl", 0x3, 2); DEFINE_CLK_RPMH_VRM(sdm845, ln_bb_clk2, ln_bb_clk2_ao, "lnbclka2", 2); @@ -217,6 +345,7 @@ DEFINE_CLK_RPMH_VRM(sdm845, ln_bb_clk3, ln_bb_clk3_ao, "lnbclka3", 2); DEFINE_CLK_RPMH_VRM(sdm845, rf_clk1, rf_clk1_ao, "rfclka1", 1); DEFINE_CLK_RPMH_VRM(sdm845, rf_clk2, rf_clk2_ao, "rfclka2", 1); DEFINE_CLK_RPMH_VRM(sdm845, rf_clk3, rf_clk3_ao, "rfclka3", 1); +DEFINE_CLK_RPMH_BCM(sdm845, ipa, "IP0"); static struct clk_hw *sdm845_rpmh_clocks[] = { [RPMH_CXO_CLK] = &sdm845_bi_tcxo.hw, @@ -231,6 +360,7 @@ static struct clk_hw *sdm845_rpmh_clocks[] = { [RPMH_RF_CLK2_A] = &sdm845_rf_clk2_ao.hw, [RPMH_RF_CLK3] = &sdm845_rf_clk3.hw, [RPMH_RF_CLK3_A] = &sdm845_rf_clk3_ao.hw, + [RPMH_IPA_CLK] = &sdm845_ipa.hw, }; static const struct clk_rpmh_desc clk_rpmh_sdm845 = { @@ -267,6 +397,8 @@ static int clk_rpmh_probe(struct platform_device *pdev) for (i = 0; i < desc->num_clks; i++) { u32 res_addr; + size_t aux_data_len; + const struct bcm_db *data; rpmh_clk = to_clk_rpmh(hw_clks[i]); res_addr = cmd_db_read_addr(rpmh_clk->res_name); @@ -275,6 +407,20 @@ static int clk_rpmh_probe(struct platform_device *pdev) rpmh_clk->res_name); return -ENODEV; } + + data = cmd_db_read_aux_data(rpmh_clk->res_name, &aux_data_len); + if (IS_ERR(data)) { + ret = PTR_ERR(data); + dev_err(&pdev->dev, + "error reading RPMh aux data for %s (%d)\n", + rpmh_clk->res_name, ret); + return ret; + } + + /* Convert unit from Khz to Hz */ + if (aux_data_len == sizeof(*data)) + rpmh_clk->unit = le32_to_cpu(data->unit) * 1000ULL; + rpmh_clk->res_addr += res_addr; rpmh_clk->dev = &pdev->dev; diff --git a/drivers/clk/qcom/clk-smd-rpm.c b/drivers/clk/qcom/clk-smd-rpm.c index d3aadaeb29033..22dd42ad92236 100644 --- a/drivers/clk/qcom/clk-smd-rpm.c +++ b/drivers/clk/qcom/clk-smd-rpm.c @@ -655,10 +655,73 @@ static const struct rpm_smd_clk_desc rpm_clk_qcs404 = { .num_clks = ARRAY_SIZE(qcs404_clks), }; +/* msm8998 */ +DEFINE_CLK_SMD_RPM(msm8998, snoc_clk, snoc_a_clk, QCOM_SMD_RPM_BUS_CLK, 1); +DEFINE_CLK_SMD_RPM(msm8998, cnoc_clk, cnoc_a_clk, QCOM_SMD_RPM_BUS_CLK, 2); +DEFINE_CLK_SMD_RPM(msm8998, ce1_clk, ce1_a_clk, QCOM_SMD_RPM_CE_CLK, 0); +DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8998, div_clk1, div_clk1_a, 0xb); +DEFINE_CLK_SMD_RPM(msm8998, ipa_clk, ipa_a_clk, QCOM_SMD_RPM_IPA_CLK, 0); +DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8998, ln_bb_clk1, ln_bb_clk1_a, 1); +DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8998, ln_bb_clk2, ln_bb_clk2_a, 2); +DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8998, ln_bb_clk3_pin, ln_bb_clk3_a_pin, + 3); +DEFINE_CLK_SMD_RPM(msm8998, mmssnoc_axi_rpm_clk, mmssnoc_axi_rpm_a_clk, + QCOM_SMD_RPM_MMAXI_CLK, 0); +DEFINE_CLK_SMD_RPM(msm8998, aggre1_noc_clk, aggre1_noc_a_clk, + QCOM_SMD_RPM_AGGR_CLK, 1); +DEFINE_CLK_SMD_RPM(msm8998, aggre2_noc_clk, aggre2_noc_a_clk, + QCOM_SMD_RPM_AGGR_CLK, 2); +DEFINE_CLK_SMD_RPM_QDSS(msm8998, qdss_clk, qdss_a_clk, + QCOM_SMD_RPM_MISC_CLK, 1); +DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8998, rf_clk1, rf_clk1_a, 4); +DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8998, rf_clk2_pin, rf_clk2_a_pin, 5); +DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8998, rf_clk3, rf_clk3_a, 6); +DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8998, rf_clk3_pin, rf_clk3_a_pin, 6); +static struct clk_smd_rpm *msm8998_clks[] = { + [RPM_SMD_SNOC_CLK] = &msm8998_snoc_clk, + [RPM_SMD_SNOC_A_CLK] = &msm8998_snoc_a_clk, + [RPM_SMD_CNOC_CLK] = &msm8998_cnoc_clk, + [RPM_SMD_CNOC_A_CLK] = &msm8998_cnoc_a_clk, + [RPM_SMD_CE1_CLK] = &msm8998_ce1_clk, + [RPM_SMD_CE1_A_CLK] = &msm8998_ce1_a_clk, + [RPM_SMD_DIV_CLK1] = &msm8998_div_clk1, + [RPM_SMD_DIV_A_CLK1] = &msm8998_div_clk1_a, + [RPM_SMD_IPA_CLK] = &msm8998_ipa_clk, + [RPM_SMD_IPA_A_CLK] = &msm8998_ipa_a_clk, + [RPM_SMD_LN_BB_CLK1] = &msm8998_ln_bb_clk1, + [RPM_SMD_LN_BB_CLK1_A] = &msm8998_ln_bb_clk1_a, + [RPM_SMD_LN_BB_CLK2] = &msm8998_ln_bb_clk2, + [RPM_SMD_LN_BB_CLK2_A] = &msm8998_ln_bb_clk2_a, + [RPM_SMD_LN_BB_CLK3_PIN] = &msm8998_ln_bb_clk3_pin, + [RPM_SMD_LN_BB_CLK3_A_PIN] = &msm8998_ln_bb_clk3_a_pin, + [RPM_SMD_MMAXI_CLK] = &msm8998_mmssnoc_axi_rpm_clk, + [RPM_SMD_MMAXI_A_CLK] = &msm8998_mmssnoc_axi_rpm_a_clk, + [RPM_SMD_AGGR1_NOC_CLK] = &msm8998_aggre1_noc_clk, + [RPM_SMD_AGGR1_NOC_A_CLK] = &msm8998_aggre1_noc_a_clk, + [RPM_SMD_AGGR2_NOC_CLK] = &msm8998_aggre2_noc_clk, + [RPM_SMD_AGGR2_NOC_A_CLK] = &msm8998_aggre2_noc_a_clk, + [RPM_SMD_QDSS_CLK] = &msm8998_qdss_clk, + [RPM_SMD_QDSS_A_CLK] = &msm8998_qdss_a_clk, + [RPM_SMD_RF_CLK1] = &msm8998_rf_clk1, + [RPM_SMD_RF_CLK1_A] = &msm8998_rf_clk1_a, + [RPM_SMD_RF_CLK2_PIN] = &msm8998_rf_clk2_pin, + [RPM_SMD_RF_CLK2_A_PIN] = &msm8998_rf_clk2_a_pin, + [RPM_SMD_RF_CLK3] = &msm8998_rf_clk3, + [RPM_SMD_RF_CLK3_A] = &msm8998_rf_clk3_a, + [RPM_SMD_RF_CLK3_PIN] = &msm8998_rf_clk3_pin, + [RPM_SMD_RF_CLK3_A_PIN] = &msm8998_rf_clk3_a_pin, +}; + +static const struct rpm_smd_clk_desc rpm_clk_msm8998 = { + .clks = msm8998_clks, + .num_clks = ARRAY_SIZE(msm8998_clks), +}; + static const struct of_device_id rpm_smd_clk_match_table[] = { { .compatible = "qcom,rpmcc-msm8916", .data = &rpm_clk_msm8916 }, { .compatible = "qcom,rpmcc-msm8974", .data = &rpm_clk_msm8974 }, { .compatible = "qcom,rpmcc-msm8996", .data = &rpm_clk_msm8996 }, + { .compatible = "qcom,rpmcc-msm8998", .data = &rpm_clk_msm8998 }, { .compatible = "qcom,rpmcc-qcs404", .data = &rpm_clk_qcs404 }, { } }; diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c index 0a48ed56833b4..a6b2f86112d86 100644 --- a/drivers/clk/qcom/common.c +++ b/drivers/clk/qcom/common.c @@ -231,6 +231,8 @@ int qcom_cc_really_probe(struct platform_device *pdev, struct gdsc_desc *scd; size_t num_clks = desc->num_clks; struct clk_regmap **rclks = desc->clks; + size_t num_clk_hws = desc->num_clk_hws; + struct clk_hw **clk_hws = desc->clk_hws; cc = devm_kzalloc(dev, sizeof(*cc), GFP_KERNEL); if (!cc) @@ -269,6 +271,12 @@ int qcom_cc_really_probe(struct platform_device *pdev, qcom_cc_drop_protected(dev, cc); + for (i = 0; i < num_clk_hws; i++) { + ret = devm_clk_hw_register(dev, clk_hws[i]); + if (ret) + return ret; + } + for (i = 0; i < num_clks; i++) { if (!rclks[i]) continue; diff --git a/drivers/clk/qcom/common.h b/drivers/clk/qcom/common.h index 4aa33ee70bae1..1e2a8bdac55a3 100644 --- a/drivers/clk/qcom/common.h +++ b/drivers/clk/qcom/common.h @@ -27,6 +27,8 @@ struct qcom_cc_desc { size_t num_resets; struct gdsc **gdscs; size_t num_gdscs; + struct clk_hw **clk_hws; + size_t num_clk_hws; }; /** diff --git a/drivers/clk/qcom/gcc-ipq8074.c b/drivers/clk/qcom/gcc-ipq8074.c index 505c6263141d9..0e32892b438c7 100644 --- a/drivers/clk/qcom/gcc-ipq8074.c +++ b/drivers/clk/qcom/gcc-ipq8074.c @@ -4715,18 +4715,12 @@ static const struct qcom_cc_desc gcc_ipq8074_desc = { .num_clks = ARRAY_SIZE(gcc_ipq8074_clks), .resets = gcc_ipq8074_resets, .num_resets = ARRAY_SIZE(gcc_ipq8074_resets), + .clk_hws = gcc_ipq8074_hws, + .num_clk_hws = ARRAY_SIZE(gcc_ipq8074_hws), }; static int gcc_ipq8074_probe(struct platform_device *pdev) { - int ret, i; - - for (i = 0; i < ARRAY_SIZE(gcc_ipq8074_hws); i++) { - ret = devm_clk_hw_register(&pdev->dev, gcc_ipq8074_hws[i]); - if (ret) - return ret; - } - return qcom_cc_probe(pdev, &gcc_ipq8074_desc); } diff --git a/drivers/clk/qcom/gcc-mdm9615.c b/drivers/clk/qcom/gcc-mdm9615.c index 849046fbed6d4..8c6d93144b9ce 100644 --- a/drivers/clk/qcom/gcc-mdm9615.c +++ b/drivers/clk/qcom/gcc-mdm9615.c @@ -1702,6 +1702,8 @@ static const struct qcom_cc_desc gcc_mdm9615_desc = { .num_clks = ARRAY_SIZE(gcc_mdm9615_clks), .resets = gcc_mdm9615_resets, .num_resets = ARRAY_SIZE(gcc_mdm9615_resets), + .clk_hws = gcc_mdm9615_hws, + .num_clk_hws = ARRAY_SIZE(gcc_mdm9615_hws), }; static const struct of_device_id gcc_mdm9615_match_table[] = { @@ -1712,21 +1714,12 @@ MODULE_DEVICE_TABLE(of, gcc_mdm9615_match_table); static int gcc_mdm9615_probe(struct platform_device *pdev) { - struct device *dev = &pdev->dev; struct regmap *regmap; - int ret; - int i; regmap = qcom_cc_map(pdev, &gcc_mdm9615_desc); if (IS_ERR(regmap)) return PTR_ERR(regmap); - for (i = 0; i < ARRAY_SIZE(gcc_mdm9615_hws); i++) { - ret = devm_clk_hw_register(dev, gcc_mdm9615_hws[i]); - if (ret) - return ret; - } - return qcom_cc_really_probe(pdev, &gcc_mdm9615_desc, regmap); } diff --git a/drivers/clk/qcom/gcc-msm8996.c b/drivers/clk/qcom/gcc-msm8996.c index 9d136172c27ca..4632b9272b7f4 100644 --- a/drivers/clk/qcom/gcc-msm8996.c +++ b/drivers/clk/qcom/gcc-msm8996.c @@ -3656,6 +3656,8 @@ static const struct qcom_cc_desc gcc_msm8996_desc = { .num_resets = ARRAY_SIZE(gcc_msm8996_resets), .gdscs = gcc_msm8996_gdscs, .num_gdscs = ARRAY_SIZE(gcc_msm8996_gdscs), + .clk_hws = gcc_msm8996_hws, + .num_clk_hws = ARRAY_SIZE(gcc_msm8996_hws), }; static const struct of_device_id gcc_msm8996_match_table[] = { @@ -3666,8 +3668,6 @@ MODULE_DEVICE_TABLE(of, gcc_msm8996_match_table); static int gcc_msm8996_probe(struct platform_device *pdev) { - struct device *dev = &pdev->dev; - int i, ret; struct regmap *regmap; regmap = qcom_cc_map(pdev, &gcc_msm8996_desc); @@ -3680,12 +3680,6 @@ static int gcc_msm8996_probe(struct platform_device *pdev) */ regmap_update_bits(regmap, 0x52008, BIT(21), BIT(21)); - for (i = 0; i < ARRAY_SIZE(gcc_msm8996_hws); i++) { - ret = devm_clk_hw_register(dev, gcc_msm8996_hws[i]); - if (ret) - return ret; - } - return qcom_cc_really_probe(pdev, &gcc_msm8996_desc, regmap); } diff --git a/drivers/clk/qcom/gcc-msm8998.c b/drivers/clk/qcom/gcc-msm8998.c index 1b779396e04fa..c240fba794c7a 100644 --- a/drivers/clk/qcom/gcc-msm8998.c +++ b/drivers/clk/qcom/gcc-msm8998.c @@ -1112,6 +1112,7 @@ static struct clk_rcg2 ufs_axi_clk_src = { static const struct freq_tbl ftbl_usb30_master_clk_src[] = { F(19200000, P_XO, 1, 0, 0), + F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0), F(120000000, P_GPLL0_OUT_MAIN, 5, 0, 0), F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), { } @@ -1189,6 +1190,7 @@ static struct clk_branch gcc_aggre1_ufs_axi_clk = { "ufs_axi_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1206,6 +1208,7 @@ static struct clk_branch gcc_aggre1_usb3_axi_clk = { "usb30_master_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1288,6 +1291,7 @@ static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = { "blsp1_qup1_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1305,6 +1309,7 @@ static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = { "blsp1_qup1_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1322,6 +1327,7 @@ static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = { "blsp1_qup2_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1339,6 +1345,7 @@ static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = { "blsp1_qup2_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1356,6 +1363,7 @@ static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = { "blsp1_qup3_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1373,6 +1381,7 @@ static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = { "blsp1_qup3_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1390,6 +1399,7 @@ static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = { "blsp1_qup4_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1407,6 +1417,7 @@ static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = { "blsp1_qup4_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1424,6 +1435,7 @@ static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = { "blsp1_qup5_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1441,6 +1453,7 @@ static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = { "blsp1_qup5_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1458,6 +1471,7 @@ static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = { "blsp1_qup6_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1475,6 +1489,7 @@ static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = { "blsp1_qup6_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1505,6 +1520,7 @@ static struct clk_branch gcc_blsp1_uart1_apps_clk = { "blsp1_uart1_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1522,6 +1538,7 @@ static struct clk_branch gcc_blsp1_uart2_apps_clk = { "blsp1_uart2_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1539,6 +1556,7 @@ static struct clk_branch gcc_blsp1_uart3_apps_clk = { "blsp1_uart3_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1569,6 +1587,7 @@ static struct clk_branch gcc_blsp2_qup1_i2c_apps_clk = { "blsp2_qup1_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1586,6 +1605,7 @@ static struct clk_branch gcc_blsp2_qup1_spi_apps_clk = { "blsp2_qup1_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1603,6 +1623,7 @@ static struct clk_branch gcc_blsp2_qup2_i2c_apps_clk = { "blsp2_qup2_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1620,6 +1641,7 @@ static struct clk_branch gcc_blsp2_qup2_spi_apps_clk = { "blsp2_qup2_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1637,6 +1659,7 @@ static struct clk_branch gcc_blsp2_qup3_i2c_apps_clk = { "blsp2_qup3_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1654,6 +1677,7 @@ static struct clk_branch gcc_blsp2_qup3_spi_apps_clk = { "blsp2_qup3_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1671,6 +1695,7 @@ static struct clk_branch gcc_blsp2_qup4_i2c_apps_clk = { "blsp2_qup4_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1688,6 +1713,7 @@ static struct clk_branch gcc_blsp2_qup4_spi_apps_clk = { "blsp2_qup4_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1705,6 +1731,7 @@ static struct clk_branch gcc_blsp2_qup5_i2c_apps_clk = { "blsp2_qup5_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1722,6 +1749,7 @@ static struct clk_branch gcc_blsp2_qup5_spi_apps_clk = { "blsp2_qup5_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1739,6 +1767,7 @@ static struct clk_branch gcc_blsp2_qup6_i2c_apps_clk = { "blsp2_qup6_i2c_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1756,6 +1785,7 @@ static struct clk_branch gcc_blsp2_qup6_spi_apps_clk = { "blsp2_qup6_spi_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1786,6 +1816,7 @@ static struct clk_branch gcc_blsp2_uart1_apps_clk = { "blsp2_uart1_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1803,6 +1834,7 @@ static struct clk_branch gcc_blsp2_uart2_apps_clk = { "blsp2_uart2_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1820,6 +1852,7 @@ static struct clk_branch gcc_blsp2_uart3_apps_clk = { "blsp2_uart3_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1837,6 +1870,7 @@ static struct clk_branch gcc_cfg_noc_usb3_axi_clk = { "usb30_master_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1854,6 +1888,7 @@ static struct clk_branch gcc_gp1_clk = { "gp1_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1871,6 +1906,7 @@ static struct clk_branch gcc_gp2_clk = { "gp2_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1888,6 +1924,7 @@ static struct clk_branch gcc_gp3_clk = { "gp3_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1957,6 +1994,7 @@ static struct clk_branch gcc_hmss_ahb_clk = { "hmss_ahb_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1987,6 +2025,7 @@ static struct clk_branch gcc_hmss_rbcpr_clk = { "hmss_rbcpr_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2088,6 +2127,7 @@ static struct clk_branch gcc_pcie_0_aux_clk = { "pcie_aux_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2157,6 +2197,7 @@ static struct clk_branch gcc_pcie_phy_aux_clk = { "pcie_aux_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2174,6 +2215,7 @@ static struct clk_branch gcc_pdm2_clk = { "pdm2_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2243,6 +2285,7 @@ static struct clk_branch gcc_sdcc2_apps_clk = { "sdcc2_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2273,6 +2316,7 @@ static struct clk_branch gcc_sdcc4_apps_clk = { "sdcc4_apps_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2316,6 +2360,7 @@ static struct clk_branch gcc_tsif_ref_clk = { "tsif_ref_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2346,6 +2391,7 @@ static struct clk_branch gcc_ufs_axi_clk = { "ufs_axi_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2441,6 +2487,7 @@ static struct clk_branch gcc_usb30_master_clk = { "usb30_master_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2458,6 +2505,7 @@ static struct clk_branch gcc_usb30_mock_utmi_clk = { "usb30_mock_utmi_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2488,6 +2536,7 @@ static struct clk_branch gcc_usb3_phy_aux_clk = { "usb3_phy_aux_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2495,7 +2544,7 @@ static struct clk_branch gcc_usb3_phy_aux_clk = { static struct clk_branch gcc_usb3_phy_pipe_clk = { .halt_reg = 0x50004, - .halt_check = BRANCH_HALT, + .halt_check = BRANCH_HALT_SKIP, .clkr = { .enable_reg = 0x50004, .enable_mask = BIT(0), @@ -2910,6 +2959,10 @@ static const struct regmap_config gcc_msm8998_regmap_config = { .fast_io = true, }; +static struct clk_hw *gcc_msm8998_hws[] = { + &xo.hw, +}; + static const struct qcom_cc_desc gcc_msm8998_desc = { .config = &gcc_msm8998_regmap_config, .clks = gcc_msm8998_clocks, @@ -2918,6 +2971,8 @@ static const struct qcom_cc_desc gcc_msm8998_desc = { .num_resets = ARRAY_SIZE(gcc_msm8998_resets), .gdscs = gcc_msm8998_gdscs, .num_gdscs = ARRAY_SIZE(gcc_msm8998_gdscs), + .clk_hws = gcc_msm8998_hws, + .num_clk_hws = ARRAY_SIZE(gcc_msm8998_hws), }; static int gcc_msm8998_probe(struct platform_device *pdev) @@ -2937,10 +2992,6 @@ static int gcc_msm8998_probe(struct platform_device *pdev) if (ret) return ret; - ret = devm_clk_hw_register(&pdev->dev, &xo.hw); - if (ret) - return ret; - return qcom_cc_really_probe(pdev, &gcc_msm8998_desc, regmap); } diff --git a/drivers/clk/qcom/gcc-qcs404.c b/drivers/clk/qcom/gcc-qcs404.c index 64da032bb9edb..5a62f64ada930 100644 --- a/drivers/clk/qcom/gcc-qcs404.c +++ b/drivers/clk/qcom/gcc-qcs404.c @@ -678,6 +678,7 @@ static struct clk_rcg2 blsp1_uart3_apps_clk_src = { .cmd_rcgr = 0x4014, .mnd_width = 16, .hid_width = 5, + .cfg_off = 0x20, .parent_map = gcc_parent_map_0, .freq_tbl = ftbl_blsp1_uart0_apps_clk_src, .clkr.hw.init = &(struct clk_init_data){ @@ -2692,6 +2693,8 @@ static const struct qcom_cc_desc gcc_qcs404_desc = { .num_clks = ARRAY_SIZE(gcc_qcs404_clocks), .resets = gcc_qcs404_resets, .num_resets = ARRAY_SIZE(gcc_qcs404_resets), + .clk_hws = gcc_qcs404_hws, + .num_clk_hws = ARRAY_SIZE(gcc_qcs404_hws), }; static const struct of_device_id gcc_qcs404_match_table[] = { @@ -2703,7 +2706,6 @@ MODULE_DEVICE_TABLE(of, gcc_qcs404_match_table); static int gcc_qcs404_probe(struct platform_device *pdev) { struct regmap *regmap; - int ret, i; regmap = qcom_cc_map(pdev, &gcc_qcs404_desc); if (IS_ERR(regmap)) @@ -2711,12 +2713,6 @@ static int gcc_qcs404_probe(struct platform_device *pdev) clk_alpha_pll_configure(&gpll3_out_main, regmap, &gpll3_config); - for (i = 0; i < ARRAY_SIZE(gcc_qcs404_hws); i++) { - ret = devm_clk_hw_register(&pdev->dev, gcc_qcs404_hws[i]); - if (ret) - return ret; - } - return qcom_cc_really_probe(pdev, &gcc_qcs404_desc, regmap); } diff --git a/drivers/clk/qcom/gcc-sdm660.c b/drivers/clk/qcom/gcc-sdm660.c index ba239ea4c842a..8827db23066f5 100644 --- a/drivers/clk/qcom/gcc-sdm660.c +++ b/drivers/clk/qcom/gcc-sdm660.c @@ -2420,6 +2420,8 @@ static const struct qcom_cc_desc gcc_sdm660_desc = { .num_resets = ARRAY_SIZE(gcc_sdm660_resets), .gdscs = gcc_sdm660_gdscs, .num_gdscs = ARRAY_SIZE(gcc_sdm660_gdscs), + .clk_hws = gcc_sdm660_hws, + .num_clk_hws = ARRAY_SIZE(gcc_sdm660_hws), }; static const struct of_device_id gcc_sdm660_match_table[] = { @@ -2431,7 +2433,7 @@ MODULE_DEVICE_TABLE(of, gcc_sdm660_match_table); static int gcc_sdm660_probe(struct platform_device *pdev) { - int i, ret; + int ret; struct regmap *regmap; regmap = qcom_cc_map(pdev, &gcc_sdm660_desc); @@ -2446,13 +2448,6 @@ static int gcc_sdm660_probe(struct platform_device *pdev) if (ret) return ret; - /* Register the hws */ - for (i = 0; i < ARRAY_SIZE(gcc_sdm660_hws); i++) { - ret = devm_clk_hw_register(&pdev->dev, gcc_sdm660_hws[i]); - if (ret) - return ret; - } - return qcom_cc_really_probe(pdev, &gcc_sdm660_desc, regmap); } diff --git a/drivers/clk/qcom/gcc-sdm845.c b/drivers/clk/qcom/gcc-sdm845.c index 58fa5c247af10..7131dcf9b0603 100644 --- a/drivers/clk/qcom/gcc-sdm845.c +++ b/drivers/clk/qcom/gcc-sdm845.c @@ -1703,6 +1703,9 @@ static struct clk_branch gcc_pcie_0_pipe_clk = { .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "gcc_pcie_0_pipe_clk", + .parent_names = (const char *[]){ "pcie_0_pipe_clk" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1802,6 +1805,8 @@ static struct clk_branch gcc_pcie_1_pipe_clk = { .enable_mask = BIT(30), .hw.init = &(struct clk_init_data){ .name = "gcc_pcie_1_pipe_clk", + .parent_names = (const char *[]){ "pcie_1_pipe_clk" }, + .num_parents = 1, .ops = &clk_branch2_ops, }, }, diff --git a/drivers/clk/qcom/mmcc-msm8996.c b/drivers/clk/qcom/mmcc-msm8996.c index 7d4ee109435c3..7235510eac944 100644 --- a/drivers/clk/qcom/mmcc-msm8996.c +++ b/drivers/clk/qcom/mmcc-msm8996.c @@ -3347,6 +3347,8 @@ static const struct qcom_cc_desc mmcc_msm8996_desc = { .num_resets = ARRAY_SIZE(mmcc_msm8996_resets), .gdscs = mmcc_msm8996_gdscs, .num_gdscs = ARRAY_SIZE(mmcc_msm8996_gdscs), + .clk_hws = mmcc_msm8996_hws, + .num_clk_hws = ARRAY_SIZE(mmcc_msm8996_hws), }; static const struct of_device_id mmcc_msm8996_match_table[] = { @@ -3357,8 +3359,6 @@ MODULE_DEVICE_TABLE(of, mmcc_msm8996_match_table); static int mmcc_msm8996_probe(struct platform_device *pdev) { - struct device *dev = &pdev->dev; - int i, ret; struct regmap *regmap; regmap = qcom_cc_map(pdev, &mmcc_msm8996_desc); @@ -3370,12 +3370,6 @@ static int mmcc_msm8996_probe(struct platform_device *pdev) /* Disable the NoC FSM for mmss_mmagic_cfg_ahb_clk */ regmap_update_bits(regmap, 0x5054, BIT(15), 0); - for (i = 0; i < ARRAY_SIZE(mmcc_msm8996_hws); i++) { - ret = devm_clk_hw_register(dev, mmcc_msm8996_hws[i]); - if (ret) - return ret; - } - return qcom_cc_really_probe(pdev, &mmcc_msm8996_desc, regmap); } diff --git a/drivers/clk/renesas/r8a774a1-cpg-mssr.c b/drivers/clk/renesas/r8a774a1-cpg-mssr.c index 10e852518870c..4d92b27a61538 100644 --- a/drivers/clk/renesas/r8a774a1-cpg-mssr.c +++ b/drivers/clk/renesas/r8a774a1-cpg-mssr.c @@ -21,7 +21,7 @@ enum clk_ids { /* Core Clock Outputs exported to DT */ - LAST_DT_CORE_CLK = R8A774A1_CLK_OSC, + LAST_DT_CORE_CLK = R8A774A1_CLK_CANFD, /* External Input Clocks */ CLK_EXTAL, @@ -102,6 +102,7 @@ static const struct cpg_core_clk r8a774a1_core_clks[] __initconst = { DEF_FIXED("cp", R8A774A1_CLK_CP, CLK_EXTAL, 2, 1), DEF_FIXED("cpex", R8A774A1_CLK_CPEX, CLK_EXTAL, 2, 1), + DEF_DIV6P1("canfd", R8A774A1_CLK_CANFD, CLK_PLL1_DIV4, 0x244), DEF_DIV6P1("csi0", R8A774A1_CLK_CSI0, CLK_PLL1_DIV4, 0x00c), DEF_DIV6P1("mso", R8A774A1_CLK_MSO, CLK_PLL1_DIV4, 0x014), DEF_DIV6P1("hdmi", R8A774A1_CLK_HDMI, CLK_PLL1_DIV4, 0x250), @@ -191,6 +192,7 @@ static const struct mssr_mod_clk r8a774a1_mod_clks[] __initconst = { DEF_MOD("gpio2", 910, R8A774A1_CLK_S3D4), DEF_MOD("gpio1", 911, R8A774A1_CLK_S3D4), DEF_MOD("gpio0", 912, R8A774A1_CLK_S3D4), + DEF_MOD("can-fd", 914, R8A774A1_CLK_S3D2), DEF_MOD("can-if1", 915, R8A774A1_CLK_S3D4), DEF_MOD("can-if0", 916, R8A774A1_CLK_S3D4), DEF_MOD("i2c6", 918, R8A774A1_CLK_S0D6), diff --git a/drivers/clk/renesas/r8a774c0-cpg-mssr.c b/drivers/clk/renesas/r8a774c0-cpg-mssr.c index 10b96895d4521..34e274f2a273a 100644 --- a/drivers/clk/renesas/r8a774c0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a774c0-cpg-mssr.c @@ -22,7 +22,7 @@ enum clk_ids { /* Core Clock Outputs exported to DT */ - LAST_DT_CORE_CLK = R8A774C0_CLK_CPEX, + LAST_DT_CORE_CLK = R8A774C0_CLK_CANFD, /* External Input Clocks */ CLK_EXTAL, @@ -33,6 +33,7 @@ enum clk_ids { CLK_PLL1, CLK_PLL3, CLK_PLL0D4, + CLK_PLL0D6, CLK_PLL0D8, CLK_PLL0D20, CLK_PLL0D24, @@ -61,6 +62,7 @@ static const struct cpg_core_clk r8a774c0_core_clks[] __initconst = { DEF_FIXED(".pll0", CLK_PLL0, CLK_MAIN, 1, 100), DEF_FIXED(".pll0d4", CLK_PLL0D4, CLK_PLL0, 4, 1), + DEF_FIXED(".pll0d6", CLK_PLL0D6, CLK_PLL0, 6, 1), DEF_FIXED(".pll0d8", CLK_PLL0D8, CLK_PLL0, 8, 1), DEF_FIXED(".pll0d20", CLK_PLL0D20, CLK_PLL0, 20, 1), DEF_FIXED(".pll0d24", CLK_PLL0D24, CLK_PLL0, 24, 1), @@ -112,6 +114,7 @@ static const struct cpg_core_clk r8a774c0_core_clks[] __initconst = { DEF_GEN3_PE("s3d2c", R8A774C0_CLK_S3D2C, CLK_S3, 2, CLK_PE, 2), DEF_GEN3_PE("s3d4c", R8A774C0_CLK_S3D4C, CLK_S3, 4, CLK_PE, 4), + DEF_DIV6P1("canfd", R8A774C0_CLK_CANFD, CLK_PLL0D6, 0x244), DEF_DIV6P1("csi0", R8A774C0_CLK_CSI0, CLK_PLL1D2, 0x00c), DEF_DIV6P1("mso", R8A774C0_CLK_MSO, CLK_PLL1D2, 0x014), @@ -119,6 +122,11 @@ static const struct cpg_core_clk r8a774c0_core_clks[] __initconst = { }; static const struct mssr_mod_clk r8a774c0_mod_clks[] __initconst = { + DEF_MOD("tmu4", 121, R8A774C0_CLK_S0D6C), + DEF_MOD("tmu3", 122, R8A774C0_CLK_S3D2C), + DEF_MOD("tmu2", 123, R8A774C0_CLK_S3D2C), + DEF_MOD("tmu1", 124, R8A774C0_CLK_S3D2C), + DEF_MOD("tmu0", 125, R8A774C0_CLK_CP), DEF_MOD("scif5", 202, R8A774C0_CLK_S3D4C), DEF_MOD("scif4", 203, R8A774C0_CLK_S3D4C), DEF_MOD("scif3", 204, R8A774C0_CLK_S3D4C), @@ -172,8 +180,8 @@ static const struct mssr_mod_clk r8a774c0_mod_clks[] __initconst = { DEF_MOD("ehci0", 703, R8A774C0_CLK_S3D4), DEF_MOD("hsusb", 704, R8A774C0_CLK_S3D4), DEF_MOD("csi40", 716, R8A774C0_CLK_CSI0), - DEF_MOD("du1", 723, R8A774C0_CLK_S2D1), - DEF_MOD("du0", 724, R8A774C0_CLK_S2D1), + DEF_MOD("du1", 723, R8A774C0_CLK_S1D1), + DEF_MOD("du0", 724, R8A774C0_CLK_S1D1), DEF_MOD("lvds", 727, R8A774C0_CLK_S2D1), DEF_MOD("vin5", 806, R8A774C0_CLK_S1D2), @@ -187,6 +195,7 @@ static const struct mssr_mod_clk r8a774c0_mod_clks[] __initconst = { DEF_MOD("gpio2", 910, R8A774C0_CLK_S3D4), DEF_MOD("gpio1", 911, R8A774C0_CLK_S3D4), DEF_MOD("gpio0", 912, R8A774C0_CLK_S3D4), + DEF_MOD("can-fd", 914, R8A774C0_CLK_S3D2), DEF_MOD("can-if1", 915, R8A774C0_CLK_S3D4), DEF_MOD("can-if0", 916, R8A774C0_CLK_S3D4), DEF_MOD("i2c6", 918, R8A774C0_CLK_S3D2), diff --git a/drivers/clk/renesas/r8a77980-cpg-mssr.c b/drivers/clk/renesas/r8a77980-cpg-mssr.c index 25a3083b67641..f9e07fcc0d96f 100644 --- a/drivers/clk/renesas/r8a77980-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77980-cpg-mssr.c @@ -41,6 +41,7 @@ enum clk_ids { CLK_S2, CLK_S3, CLK_SDSRC, + CLK_RPCSRC, CLK_OCO, /* Module Clocks */ @@ -65,8 +66,14 @@ static const struct cpg_core_clk r8a77980_core_clks[] __initconst = { DEF_FIXED(".s2", CLK_S2, CLK_PLL1_DIV2, 4, 1), DEF_FIXED(".s3", CLK_S3, CLK_PLL1_DIV2, 6, 1), DEF_FIXED(".sdsrc", CLK_SDSRC, CLK_PLL1_DIV2, 2, 1), + DEF_BASE(".rpcsrc", CLK_RPCSRC, CLK_TYPE_GEN3_RPCSRC, CLK_PLL1), DEF_RATE(".oco", CLK_OCO, 32768), + DEF_BASE("rpc", R8A77980_CLK_RPC, CLK_TYPE_GEN3_RPC, + CLK_RPCSRC), + DEF_BASE("rpcd2", R8A77980_CLK_RPCD2, CLK_TYPE_GEN3_RPCD2, + R8A77980_CLK_RPC), + /* Core Clock Outputs */ DEF_FIXED("ztr", R8A77980_CLK_ZTR, CLK_PLL1_DIV2, 6, 1), DEF_FIXED("ztrd2", R8A77980_CLK_ZTRD2, CLK_PLL1_DIV2, 12, 1), @@ -164,6 +171,7 @@ static const struct mssr_mod_clk r8a77980_mod_clks[] __initconst = { DEF_MOD("gpio1", 911, R8A77980_CLK_CP), DEF_MOD("gpio0", 912, R8A77980_CLK_CP), DEF_MOD("can-fd", 914, R8A77980_CLK_S3D2), + DEF_MOD("rpc-if", 917, R8A77980_CLK_RPC), DEF_MOD("i2c4", 927, R8A77980_CLK_S0D6), DEF_MOD("i2c3", 928, R8A77980_CLK_S0D6), DEF_MOD("i2c2", 929, R8A77980_CLK_S3D2), diff --git a/drivers/clk/renesas/rcar-gen3-cpg.c b/drivers/clk/renesas/rcar-gen3-cpg.c index be2ccbd6d6239..9a8071a8114da 100644 --- a/drivers/clk/renesas/rcar-gen3-cpg.c +++ b/drivers/clk/renesas/rcar-gen3-cpg.c @@ -30,6 +30,21 @@ #define CPG_RCKCR_CKSEL BIT(15) /* RCLK Clock Source Select */ +static spinlock_t cpg_lock; + +static void cpg_reg_modify(void __iomem *reg, u32 clear, u32 set) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&cpg_lock, flags); + val = readl(reg); + val &= ~clear; + val |= set; + writel(val, reg); + spin_unlock_irqrestore(&cpg_lock, flags); +}; + struct cpg_simple_notifier { struct notifier_block nb; void __iomem *reg; @@ -118,7 +133,6 @@ static int cpg_z_clk_set_rate(struct clk_hw *hw, unsigned long rate, struct cpg_z_clk *zclk = to_z_clk(hw); unsigned int mult; unsigned int i; - u32 val, kick; /* Factor of 2 is for fixed divider */ mult = DIV_ROUND_CLOSEST_ULL(rate * 32ULL * 2, parent_rate); @@ -127,17 +141,14 @@ static int cpg_z_clk_set_rate(struct clk_hw *hw, unsigned long rate, if (readl(zclk->kick_reg) & CPG_FRQCRB_KICK) return -EBUSY; - val = readl(zclk->reg) & ~zclk->mask; - val |= ((32 - mult) << __ffs(zclk->mask)) & zclk->mask; - writel(val, zclk->reg); + cpg_reg_modify(zclk->reg, zclk->mask, + ((32 - mult) << __ffs(zclk->mask)) & zclk->mask); /* * Set KICK bit in FRQCRB to update hardware setting and wait for * clock change completion. */ - kick = readl(zclk->kick_reg); - kick |= CPG_FRQCRB_KICK; - writel(kick, zclk->kick_reg); + cpg_reg_modify(zclk->kick_reg, 0, CPG_FRQCRB_KICK); /* * Note: There is no HW information about the worst case latency. @@ -266,12 +277,10 @@ static const struct sd_div_table cpg_sd_div_table[] = { static int cpg_sd_clock_enable(struct clk_hw *hw) { struct sd_clock *clock = to_sd_clock(hw); - u32 val = readl(clock->csn.reg); - - val &= ~(CPG_SD_STP_MASK); - val |= clock->div_table[clock->cur_div_idx].val & CPG_SD_STP_MASK; - writel(val, clock->csn.reg); + cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK, + clock->div_table[clock->cur_div_idx].val & + CPG_SD_STP_MASK); return 0; } @@ -280,7 +289,7 @@ static void cpg_sd_clock_disable(struct clk_hw *hw) { struct sd_clock *clock = to_sd_clock(hw); - writel(readl(clock->csn.reg) | CPG_SD_STP_MASK, clock->csn.reg); + cpg_reg_modify(clock->csn.reg, 0, CPG_SD_STP_MASK); } static int cpg_sd_clock_is_enabled(struct clk_hw *hw) @@ -327,7 +336,6 @@ static int cpg_sd_clock_set_rate(struct clk_hw *hw, unsigned long rate, { struct sd_clock *clock = to_sd_clock(hw); unsigned int div = cpg_sd_clock_calc_div(clock, rate, parent_rate); - u32 val; unsigned int i; for (i = 0; i < clock->div_num; i++) @@ -339,10 +347,9 @@ static int cpg_sd_clock_set_rate(struct clk_hw *hw, unsigned long rate, clock->cur_div_idx = i; - val = readl(clock->csn.reg); - val &= ~(CPG_SD_STP_MASK | CPG_SD_FC_MASK); - val |= clock->div_table[i].val & (CPG_SD_STP_MASK | CPG_SD_FC_MASK); - writel(val, clock->csn.reg); + cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK | CPG_SD_FC_MASK, + clock->div_table[i].val & + (CPG_SD_STP_MASK | CPG_SD_FC_MASK)); return 0; } @@ -415,6 +422,92 @@ free_clock: return clk; } +struct rpc_clock { + struct clk_divider div; + struct clk_gate gate; + /* + * One notifier covers both RPC and RPCD2 clocks as they are both + * controlled by the same RPCCKCR register... + */ + struct cpg_simple_notifier csn; +}; + +static const struct clk_div_table cpg_rpcsrc_div_table[] = { + { 2, 5 }, { 3, 6 }, { 0, 0 }, +}; + +static const struct clk_div_table cpg_rpc_div_table[] = { + { 1, 2 }, { 3, 4 }, { 5, 6 }, { 7, 8 }, { 0, 0 }, +}; + +static struct clk * __init cpg_rpc_clk_register(const char *name, + void __iomem *base, const char *parent_name, + struct raw_notifier_head *notifiers) +{ + struct rpc_clock *rpc; + struct clk *clk; + + rpc = kzalloc(sizeof(*rpc), GFP_KERNEL); + if (!rpc) + return ERR_PTR(-ENOMEM); + + rpc->div.reg = base + CPG_RPCCKCR; + rpc->div.width = 3; + rpc->div.table = cpg_rpc_div_table; + rpc->div.lock = &cpg_lock; + + rpc->gate.reg = base + CPG_RPCCKCR; + rpc->gate.bit_idx = 8; + rpc->gate.flags = CLK_GATE_SET_TO_DISABLE; + rpc->gate.lock = &cpg_lock; + + rpc->csn.reg = base + CPG_RPCCKCR; + + clk = clk_register_composite(NULL, name, &parent_name, 1, NULL, NULL, + &rpc->div.hw, &clk_divider_ops, + &rpc->gate.hw, &clk_gate_ops, 0); + if (IS_ERR(clk)) { + kfree(rpc); + return clk; + } + + cpg_simple_notifier_register(notifiers, &rpc->csn); + return clk; +} + +struct rpcd2_clock { + struct clk_fixed_factor fixed; + struct clk_gate gate; +}; + +static struct clk * __init cpg_rpcd2_clk_register(const char *name, + void __iomem *base, + const char *parent_name) +{ + struct rpcd2_clock *rpcd2; + struct clk *clk; + + rpcd2 = kzalloc(sizeof(*rpcd2), GFP_KERNEL); + if (!rpcd2) + return ERR_PTR(-ENOMEM); + + rpcd2->fixed.mult = 1; + rpcd2->fixed.div = 2; + + rpcd2->gate.reg = base + CPG_RPCCKCR; + rpcd2->gate.bit_idx = 9; + rpcd2->gate.flags = CLK_GATE_SET_TO_DISABLE; + rpcd2->gate.lock = &cpg_lock; + + clk = clk_register_composite(NULL, name, &parent_name, 1, NULL, NULL, + &rpcd2->fixed.hw, &clk_fixed_factor_ops, + &rpcd2->gate.hw, &clk_gate_ops, 0); + if (IS_ERR(clk)) + kfree(rpcd2); + + return clk; +} + static const struct rcar_gen3_cpg_pll_config *cpg_pll_config __initdata; static unsigned int cpg_clk_extalr __initdata; @@ -593,6 +686,21 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev, } break; + case CLK_TYPE_GEN3_RPCSRC: + return clk_register_divider_table(NULL, core->name, + __clk_get_name(parent), 0, + base + CPG_RPCCKCR, 3, 2, 0, + cpg_rpcsrc_div_table, + &cpg_lock); + + case CLK_TYPE_GEN3_RPC: + return cpg_rpc_clk_register(core->name, base, + __clk_get_name(parent), notifiers); + + case CLK_TYPE_GEN3_RPCD2: + return cpg_rpcd2_clk_register(core->name, base, + __clk_get_name(parent)); + default: return ERR_PTR(-EINVAL); } @@ -613,5 +721,8 @@ int __init rcar_gen3_cpg_init(const struct rcar_gen3_cpg_pll_config *config, if (attr) cpg_quirks = (uintptr_t)attr->data; pr_debug("%s: mode = 0x%x quirks = 0x%x\n", __func__, mode, cpg_quirks); + + spin_lock_init(&cpg_lock); + return 0; } diff --git a/drivers/clk/renesas/rcar-gen3-cpg.h b/drivers/clk/renesas/rcar-gen3-cpg.h index f4fb6cf16688c..eac1b057455a9 100644 --- a/drivers/clk/renesas/rcar-gen3-cpg.h +++ b/drivers/clk/renesas/rcar-gen3-cpg.h @@ -23,6 +23,9 @@ enum rcar_gen3_clk_types { CLK_TYPE_GEN3_Z2, CLK_TYPE_GEN3_OSC, /* OSC EXTAL predivider and fixed divider */ CLK_TYPE_GEN3_RCKSEL, /* Select parent/divider using RCKCR.CKSEL */ + CLK_TYPE_GEN3_RPCSRC, + CLK_TYPE_GEN3_RPC, + CLK_TYPE_GEN3_RPCD2, /* SoC specific definitions start here */ CLK_TYPE_GEN3_SOC_BASE, @@ -57,6 +60,7 @@ struct rcar_gen3_cpg_pll_config { u8 osc_prediv; }; +#define CPG_RPCCKCR 0x238 #define CPG_RCKCR 0x240 struct clk *rcar_gen3_cpg_clk_register(struct device *dev, diff --git a/drivers/clk/rockchip/clk-rk3188.c b/drivers/clk/rockchip/clk-rk3188.c index 7ea20341e870b..5ecf28854876a 100644 --- a/drivers/clk/rockchip/clk-rk3188.c +++ b/drivers/clk/rockchip/clk-rk3188.c @@ -586,12 +586,12 @@ static struct rockchip_clk_branch rk3066a_clk_branches[] __initdata = { COMPOSITE(0, "dclk_lcdc0_src", mux_pll_src_cpll_gpll_p, 0, RK2928_CLKSEL_CON(27), 0, 1, MFLAGS, 8, 8, DFLAGS, RK2928_CLKGATE_CON(3), 1, GFLAGS), - MUX(DCLK_LCDC0, "dclk_lcdc0", mux_rk3066_lcdc0_p, 0, + MUX(DCLK_LCDC0, "dclk_lcdc0", mux_rk3066_lcdc0_p, CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(27), 4, 1, MFLAGS), COMPOSITE(0, "dclk_lcdc1_src", mux_pll_src_cpll_gpll_p, 0, RK2928_CLKSEL_CON(28), 0, 1, MFLAGS, 8, 8, DFLAGS, RK2928_CLKGATE_CON(3), 2, GFLAGS), - MUX(DCLK_LCDC1, "dclk_lcdc1", mux_rk3066_lcdc1_p, 0, + MUX(DCLK_LCDC1, "dclk_lcdc1", mux_rk3066_lcdc1_p, CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(28), 4, 1, MFLAGS), COMPOSITE_NOMUX(0, "cif1_pre", "cif_src", 0, diff --git a/drivers/clk/rockchip/clk-rk3328.c b/drivers/clk/rockchip/clk-rk3328.c index faa94adb2a373..65ab5c2f48b0d 100644 --- a/drivers/clk/rockchip/clk-rk3328.c +++ b/drivers/clk/rockchip/clk-rk3328.c @@ -78,17 +78,17 @@ static struct rockchip_pll_rate_table rk3328_pll_rates[] = { static struct rockchip_pll_rate_table rk3328_pll_frac_rates[] = { /* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */ - RK3036_PLL_RATE(1016064000, 3, 127, 1, 1, 0, 134217), + RK3036_PLL_RATE(1016064000, 3, 127, 1, 1, 0, 134218), /* vco = 1016064000 */ - RK3036_PLL_RATE(983040000, 24, 983, 1, 1, 0, 671088), + RK3036_PLL_RATE(983040000, 24, 983, 1, 1, 0, 671089), /* vco = 983040000 */ - RK3036_PLL_RATE(491520000, 24, 983, 2, 1, 0, 671088), + RK3036_PLL_RATE(491520000, 24, 983, 2, 1, 0, 671089), /* vco = 983040000 */ - RK3036_PLL_RATE(61440000, 6, 215, 7, 2, 0, 671088), + RK3036_PLL_RATE(61440000, 6, 215, 7, 2, 0, 671089), /* vco = 860156000 */ - RK3036_PLL_RATE(56448000, 12, 451, 4, 4, 0, 9797894), + RK3036_PLL_RATE(56448000, 12, 451, 4, 4, 0, 9797895), /* vco = 903168000 */ - RK3036_PLL_RATE(40960000, 12, 409, 4, 5, 0, 10066329), + RK3036_PLL_RATE(40960000, 12, 409, 4, 5, 0, 10066330), /* vco = 819200000 */ { /* sentinel */ }, }; diff --git a/drivers/clk/samsung/clk-exynos4.c b/drivers/clk/samsung/clk-exynos4.c index 59d4d46667ce5..54066e6508d3e 100644 --- a/drivers/clk/samsung/clk-exynos4.c +++ b/drivers/clk/samsung/clk-exynos4.c @@ -1028,6 +1028,7 @@ static unsigned long __init exynos4_get_xom(void) xom = readl(chipid_base + 8); iounmap(chipid_base); + of_node_put(np); } return xom; diff --git a/drivers/clk/samsung/clk-exynos5-subcmu.c b/drivers/clk/samsung/clk-exynos5-subcmu.c index 93306283d764d..8ae44b5db4c2f 100644 --- a/drivers/clk/samsung/clk-exynos5-subcmu.c +++ b/drivers/clk/samsung/clk-exynos5-subcmu.c @@ -136,15 +136,20 @@ static int __init exynos5_clk_register_subcmu(struct device *parent, { struct of_phandle_args genpdspec = { .np = pd_node }; struct platform_device *pdev; + int ret; + + pdev = platform_device_alloc("exynos5-subcmu", PLATFORM_DEVID_AUTO); + if (!pdev) + return -ENOMEM; - pdev = platform_device_alloc(info->pd_name, -1); pdev->dev.parent = parent; - pdev->driver_override = "exynos5-subcmu"; platform_set_drvdata(pdev, (void *)info); of_genpd_add_device(&genpdspec, &pdev->dev); - platform_device_add(pdev); + ret = platform_device_add(pdev); + if (ret) + platform_device_put(pdev); - return 0; + return ret; } static int __init exynos5_clk_probe(struct platform_device *pdev) diff --git a/drivers/clk/samsung/clk-exynos5433.c b/drivers/clk/samsung/clk-exynos5433.c index 751e2c4fb65b6..dae1c96de9339 100644 --- a/drivers/clk/samsung/clk-exynos5433.c +++ b/drivers/clk/samsung/clk-exynos5433.c @@ -559,7 +559,7 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = { /* ENABLE_ACLK_TOP */ GATE(CLK_ACLK_G3D_400, "aclk_g3d_400", "div_aclk_g3d_400", ENABLE_ACLK_TOP, 30, CLK_IS_CRITICAL, 0), - GATE(CLK_ACLK_IMEM_SSX_266, "aclk_imem_ssx_266", + GATE(CLK_ACLK_IMEM_SSSX_266, "aclk_imem_sssx_266", "div_aclk_imem_sssx_266", ENABLE_ACLK_TOP, 29, CLK_IGNORE_UNUSED, 0), GATE(CLK_ACLK_BUS0_400, "aclk_bus0_400", "div_aclk_bus0_400", @@ -568,10 +568,10 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = { GATE(CLK_ACLK_BUS1_400, "aclk_bus1_400", "div_aclk_bus1_400", ENABLE_ACLK_TOP, 25, CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, 0), - GATE(CLK_ACLK_IMEM_200, "aclk_imem_200", "div_aclk_imem_266", + GATE(CLK_ACLK_IMEM_200, "aclk_imem_200", "div_aclk_imem_200", ENABLE_ACLK_TOP, 24, CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, 0), - GATE(CLK_ACLK_IMEM_266, "aclk_imem_266", "div_aclk_imem_200", + GATE(CLK_ACLK_IMEM_266, "aclk_imem_266", "div_aclk_imem_266", ENABLE_ACLK_TOP, 23, CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0), GATE(CLK_ACLK_PERIC_66, "aclk_peric_66", "div_aclk_peric_66_b", @@ -5467,6 +5467,35 @@ static const struct samsung_cmu_info cam1_cmu_info __initconst = { .clk_name = "aclk_cam1_400", }; +/* + * Register offset definitions for CMU_IMEM + */ +#define ENABLE_ACLK_IMEM_SLIMSSS 0x080c +#define ENABLE_PCLK_IMEM_SLIMSSS 0x0908 + +static const unsigned long imem_clk_regs[] __initconst = { + ENABLE_ACLK_IMEM_SLIMSSS, + ENABLE_PCLK_IMEM_SLIMSSS, +}; + +static const struct samsung_gate_clock imem_gate_clks[] __initconst = { + /* ENABLE_ACLK_IMEM_SLIMSSS */ + GATE(CLK_ACLK_SLIMSSS, "aclk_slimsss", "aclk_imem_sssx_266", + ENABLE_ACLK_IMEM_SLIMSSS, 0, CLK_IGNORE_UNUSED, 0), + + /* ENABLE_PCLK_IMEM_SLIMSSS */ + GATE(CLK_PCLK_SLIMSSS, "pclk_slimsss", "aclk_imem_200", + ENABLE_PCLK_IMEM_SLIMSSS, 0, CLK_IGNORE_UNUSED, 0), +}; + +static const struct samsung_cmu_info imem_cmu_info __initconst = { + .gate_clks = imem_gate_clks, + .nr_gate_clks = ARRAY_SIZE(imem_gate_clks), + .nr_clk_ids = IMEM_NR_CLK, + .clk_regs = imem_clk_regs, + .nr_clk_regs = ARRAY_SIZE(imem_clk_regs), + .clk_name = "aclk_imem_200", +}; struct exynos5433_cmu_data { struct samsung_clk_reg_dump *clk_save; @@ -5655,6 +5684,9 @@ static const struct of_device_id exynos5433_cmu_of_match[] = { .compatible = "samsung,exynos5433-cmu-mscl", .data = &mscl_cmu_info, }, { + .compatible = "samsung,exynos5433-cmu-imem", + .data = &imem_cmu_info, + }, { }, }; diff --git a/drivers/clk/samsung/clk-s3c2443.c b/drivers/clk/samsung/clk-s3c2443.c index 884067e4f1a15..f38f0e24e3b64 100644 --- a/drivers/clk/samsung/clk-s3c2443.c +++ b/drivers/clk/samsung/clk-s3c2443.c @@ -389,7 +389,7 @@ void __init s3c2443_common_clk_init(struct device_node *np, unsigned long xti_f, ARRAY_SIZE(s3c2450_gates)); samsung_clk_register_alias(ctx, s3c2450_aliases, ARRAY_SIZE(s3c2450_aliases)); - /* fall through, as s3c2450 extends the s3c2416 clocks */ + /* fall through - as s3c2450 extends the s3c2416 clocks */ case S3C2416: samsung_clk_register_div(ctx, s3c2416_dividers, ARRAY_SIZE(s3c2416_dividers)); diff --git a/drivers/clk/samsung/clk.h b/drivers/clk/samsung/clk.h index c3f309d7100dd..9cfaca5fbcdbd 100644 --- a/drivers/clk/samsung/clk.h +++ b/drivers/clk/samsung/clk.h @@ -26,7 +26,7 @@ struct samsung_clk_provider { void __iomem *reg_base; struct device *dev; spinlock_t lock; - /* clk_data must be the last entry due to variable lenght 'hws' array */ + /* clk_data must be the last entry due to variable length 'hws' array */ struct clk_hw_onecell_data clk_data; }; diff --git a/drivers/clk/socfpga/clk-gate.c b/drivers/clk/socfpga/clk-gate.c index aa7a6e6a15b65..73e03328d5c5a 100644 --- a/drivers/clk/socfpga/clk-gate.c +++ b/drivers/clk/socfpga/clk-gate.c @@ -176,8 +176,7 @@ static struct clk_ops gateclk_ops = { .set_parent = socfpga_clk_set_parent, }; -static void __init __socfpga_gate_init(struct device_node *node, - const struct clk_ops *ops) +void __init socfpga_gate_init(struct device_node *node) { u32 clk_gate[2]; u32 div_reg[3]; @@ -188,12 +187,17 @@ static void __init __socfpga_gate_init(struct device_node *node, const char *clk_name = node->name; const char *parent_name[SOCFPGA_MAX_PARENTS]; struct clk_init_data init; + struct clk_ops *ops; int rc; socfpga_clk = kzalloc(sizeof(*socfpga_clk), GFP_KERNEL); if (WARN_ON(!socfpga_clk)) return; + ops = kmemdup(&gateclk_ops, sizeof(gateclk_ops), GFP_KERNEL); + if (WARN_ON(!ops)) + return; + rc = of_property_read_u32_array(node, "clk-gate", clk_gate, 2); if (rc) clk_gate[0] = 0; @@ -202,8 +206,8 @@ static void __init __socfpga_gate_init(struct device_node *node, socfpga_clk->hw.reg = clk_mgr_base_addr + clk_gate[0]; socfpga_clk->hw.bit_idx = clk_gate[1]; - gateclk_ops.enable = clk_gate_ops.enable; - gateclk_ops.disable = clk_gate_ops.disable; + ops->enable = clk_gate_ops.enable; + ops->disable = clk_gate_ops.disable; } rc = of_property_read_u32(node, "fixed-divider", &fixed_div); @@ -234,6 +238,11 @@ static void __init __socfpga_gate_init(struct device_node *node, init.flags = 0; init.num_parents = of_clk_parent_fill(node, parent_name, SOCFPGA_MAX_PARENTS); + if (init.num_parents < 2) { + ops->get_parent = NULL; + ops->set_parent = NULL; + } + init.parent_names = parent_name; socfpga_clk->hw.hw.init = &init; @@ -246,8 +255,3 @@ static void __init __socfpga_gate_init(struct device_node *node, if (WARN_ON(rc)) return; } - -void __init socfpga_gate_init(struct device_node *node) -{ - __socfpga_gate_init(node, &gateclk_ops); -} diff --git a/drivers/clk/socfpga/clk-pll-a10.c b/drivers/clk/socfpga/clk-pll-a10.c index 35fabe1a32c30..269467e8e07e1 100644 --- a/drivers/clk/socfpga/clk-pll-a10.c +++ b/drivers/clk/socfpga/clk-pll-a10.c @@ -95,6 +95,7 @@ static struct clk * __init __socfpga_pll_init(struct device_node *node, clkmgr_np = of_find_compatible_node(NULL, NULL, "altr,clk-mgr"); clk_mgr_a10_base_addr = of_iomap(clkmgr_np, 0); + of_node_put(clkmgr_np); BUG_ON(!clk_mgr_a10_base_addr); pll_clk->hw.reg = clk_mgr_a10_base_addr + reg; diff --git a/drivers/clk/socfpga/clk-pll.c b/drivers/clk/socfpga/clk-pll.c index c7f463172e4b9..b4b44e9b59011 100644 --- a/drivers/clk/socfpga/clk-pll.c +++ b/drivers/clk/socfpga/clk-pll.c @@ -100,6 +100,7 @@ static __init struct clk *__socfpga_pll_init(struct device_node *node, clkmgr_np = of_find_compatible_node(NULL, NULL, "altr,clk-mgr"); clk_mgr_base_addr = of_iomap(clkmgr_np, 0); + of_node_put(clkmgr_np); BUG_ON(!clk_mgr_base_addr); pll_clk->hw.reg = clk_mgr_base_addr + reg; diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c index a4fa2945f2302..4b5f8f4e4ab8c 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c @@ -144,7 +144,7 @@ static SUNXI_CCU_NKM_WITH_GATE_LOCK(pll_mipi_clk, "pll-mipi", 8, 4, /* N */ 4, 2, /* K */ 0, 4, /* M */ - BIT(31), /* gate */ + BIT(31) | BIT(23) | BIT(22), /* gate */ BIT(28), /* lock */ CLK_SET_RATE_UNGATE); diff --git a/drivers/clk/tegra/clk-dfll.c b/drivers/clk/tegra/clk-dfll.c index 0400e5b1d627f..1fc71baae13bc 100644 --- a/drivers/clk/tegra/clk-dfll.c +++ b/drivers/clk/tegra/clk-dfll.c @@ -1293,8 +1293,8 @@ static int attr_enable_set(void *data, u64 val) return val ? dfll_enable(td) : dfll_disable(td); } -DEFINE_SIMPLE_ATTRIBUTE(enable_fops, attr_enable_get, attr_enable_set, - "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(enable_fops, attr_enable_get, attr_enable_set, + "%llu\n"); static int attr_lock_get(void *data, u64 *val) { @@ -1310,8 +1310,7 @@ static int attr_lock_set(void *data, u64 val) return val ? dfll_lock(td) : dfll_unlock(td); } -DEFINE_SIMPLE_ATTRIBUTE(lock_fops, attr_lock_get, attr_lock_set, - "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(lock_fops, attr_lock_get, attr_lock_set, "%llu\n"); static int attr_rate_get(void *data, u64 *val) { @@ -1328,7 +1327,7 @@ static int attr_rate_set(void *data, u64 val) return dfll_request_rate(td, val); } -DEFINE_SIMPLE_ATTRIBUTE(rate_fops, attr_rate_get, attr_rate_set, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(rate_fops, attr_rate_get, attr_rate_set, "%llu\n"); static int attr_registers_show(struct seq_file *s, void *data) { @@ -1379,10 +1378,11 @@ static void dfll_debug_init(struct tegra_dfll *td) root = debugfs_create_dir("tegra_dfll_fcpu", NULL); td->debugfs_dir = root; - debugfs_create_file("enable", S_IRUGO | S_IWUSR, root, td, &enable_fops); - debugfs_create_file("lock", S_IRUGO, root, td, &lock_fops); - debugfs_create_file("rate", S_IRUGO, root, td, &rate_fops); - debugfs_create_file("registers", S_IRUGO, root, td, &attr_registers_fops); + debugfs_create_file_unsafe("enable", 0644, root, td, + &enable_fops); + debugfs_create_file_unsafe("lock", 0444, root, td, &lock_fops); + debugfs_create_file_unsafe("rate", 0444, root, td, &rate_fops); + debugfs_create_file("registers", 0444, root, td, &attr_registers_fops); } #else diff --git a/drivers/clk/ti/adpll.c b/drivers/clk/ti/adpll.c index 688e403333b91..0c210984765af 100644 --- a/drivers/clk/ti/adpll.c +++ b/drivers/clk/ti/adpll.c @@ -614,7 +614,7 @@ static int ti_adpll_init_clkout(struct ti_adpll_data *d, init.name = child_name; init.ops = ops; - init.flags = CLK_IS_BASIC; + init.flags = 0; co->hw.init = &init; parent_names[0] = __clk_get_name(clk0); parent_names[1] = __clk_get_name(clk1); diff --git a/drivers/clk/ti/apll.c b/drivers/clk/ti/apll.c index 222f68bc3f2ae..015a657d33829 100644 --- a/drivers/clk/ti/apll.c +++ b/drivers/clk/ti/apll.c @@ -165,7 +165,7 @@ static void __init omap_clk_register_apll(void *user, ad->clk_bypass = __clk_get_hw(clk); - clk = ti_clk_register(NULL, &clk_hw->hw, node->name); + clk = ti_clk_register_omap_hw(NULL, &clk_hw->hw, node->name); if (!IS_ERR(clk)) { of_clk_add_provider(node, of_clk_src_simple_get, clk); kfree(clk_hw->hw.init->parent_names); @@ -402,7 +402,7 @@ static void __init of_omap2_apll_setup(struct device_node *node) if (ret) goto cleanup; - clk = clk_register(NULL, &clk_hw->hw); + clk = ti_clk_register_omap_hw(NULL, &clk_hw->hw, node->name); if (!IS_ERR(clk)) { of_clk_add_provider(node, of_clk_src_simple_get, clk); kfree(init); diff --git a/drivers/clk/ti/autoidle.c b/drivers/clk/ti/autoidle.c index 7bb9afbe40589..1cae226759dd6 100644 --- a/drivers/clk/ti/autoidle.c +++ b/drivers/clk/ti/autoidle.c @@ -35,7 +35,44 @@ struct clk_ti_autoidle { #define AUTOIDLE_LOW 0x1 static LIST_HEAD(autoidle_clks); -static LIST_HEAD(clk_hw_omap_clocks); + +/* + * we have some non-atomic read/write + * operations behind it, so lets + * take one lock for handling autoidle + * of all clocks + */ +static DEFINE_SPINLOCK(autoidle_spinlock); + +static int _omap2_clk_deny_idle(struct clk_hw_omap *clk) +{ + if (clk->ops && clk->ops->deny_idle) { + unsigned long irqflags; + + spin_lock_irqsave(&autoidle_spinlock, irqflags); + clk->autoidle_count++; + if (clk->autoidle_count == 1) + clk->ops->deny_idle(clk); + + spin_unlock_irqrestore(&autoidle_spinlock, irqflags); + } + return 0; +} + +static int _omap2_clk_allow_idle(struct clk_hw_omap *clk) +{ + if (clk->ops && clk->ops->allow_idle) { + unsigned long irqflags; + + spin_lock_irqsave(&autoidle_spinlock, irqflags); + clk->autoidle_count--; + if (clk->autoidle_count == 0) + clk->ops->allow_idle(clk); + + spin_unlock_irqrestore(&autoidle_spinlock, irqflags); + } + return 0; +} /** * omap2_clk_deny_idle - disable autoidle on an OMAP clock @@ -45,12 +82,15 @@ static LIST_HEAD(clk_hw_omap_clocks); */ int omap2_clk_deny_idle(struct clk *clk) { - struct clk_hw_omap *c; + struct clk_hw *hw = __clk_get_hw(clk); - c = to_clk_hw_omap(__clk_get_hw(clk)); - if (c->ops && c->ops->deny_idle) - c->ops->deny_idle(c); - return 0; + if (omap2_clk_is_hw_omap(hw)) { + struct clk_hw_omap *c = to_clk_hw_omap(hw); + + return _omap2_clk_deny_idle(c); + } + + return -EINVAL; } /** @@ -61,12 +101,15 @@ int omap2_clk_deny_idle(struct clk *clk) */ int omap2_clk_allow_idle(struct clk *clk) { - struct clk_hw_omap *c; + struct clk_hw *hw = __clk_get_hw(clk); - c = to_clk_hw_omap(__clk_get_hw(clk)); - if (c->ops && c->ops->allow_idle) - c->ops->allow_idle(c); - return 0; + if (omap2_clk_is_hw_omap(hw)) { + struct clk_hw_omap *c = to_clk_hw_omap(hw); + + return _omap2_clk_allow_idle(c); + } + + return -EINVAL; } static void _allow_autoidle(struct clk_ti_autoidle *clk) @@ -168,26 +211,6 @@ int __init of_ti_clk_autoidle_setup(struct device_node *node) } /** - * omap2_init_clk_hw_omap_clocks - initialize an OMAP clock - * @hw: struct clk_hw * to initialize - * - * Add an OMAP clock @clk to the internal list of OMAP clocks. Used - * temporarily for autoidle handling, until this support can be - * integrated into the common clock framework code in some way. No - * return value. - */ -void omap2_init_clk_hw_omap_clocks(struct clk_hw *hw) -{ - struct clk_hw_omap *c; - - if (clk_hw_get_flags(hw) & CLK_IS_BASIC) - return; - - c = to_clk_hw_omap(hw); - list_add(&c->node, &clk_hw_omap_clocks); -} - -/** * omap2_clk_enable_autoidle_all - enable autoidle on all OMAP clocks that * support it * @@ -198,11 +221,11 @@ void omap2_init_clk_hw_omap_clocks(struct clk_hw *hw) */ int omap2_clk_enable_autoidle_all(void) { - struct clk_hw_omap *c; + int ret; - list_for_each_entry(c, &clk_hw_omap_clocks, node) - if (c->ops && c->ops->allow_idle) - c->ops->allow_idle(c); + ret = omap2_clk_for_each(_omap2_clk_allow_idle); + if (ret) + return ret; _clk_generic_allow_autoidle_all(); @@ -220,11 +243,11 @@ int omap2_clk_enable_autoidle_all(void) */ int omap2_clk_disable_autoidle_all(void) { - struct clk_hw_omap *c; + int ret; - list_for_each_entry(c, &clk_hw_omap_clocks, node) - if (c->ops && c->ops->deny_idle) - c->ops->deny_idle(c); + ret = omap2_clk_for_each(_omap2_clk_deny_idle); + if (ret) + return ret; _clk_generic_deny_autoidle_all(); diff --git a/drivers/clk/ti/clk.c b/drivers/clk/ti/clk.c index 5d7fb2eecce45..ba17cc5bd04bf 100644 --- a/drivers/clk/ti/clk.c +++ b/drivers/clk/ti/clk.c @@ -31,6 +31,7 @@ #undef pr_fmt #define pr_fmt(fmt) "%s: " fmt, __func__ +static LIST_HEAD(clk_hw_omap_clocks); struct ti_clk_ll_ops *ti_clk_ll_ops; static struct device_node *clocks_node_ptr[CLK_MAX_MEMMAPS]; @@ -191,9 +192,13 @@ void __init ti_dt_clocks_register(struct ti_dt_clk oclks[]) clkdev_add(&c->lk); } else { if (num_args && !has_clkctrl_data) { - if (of_find_compatible_node(NULL, NULL, - "ti,clkctrl")) { + struct device_node *np; + + np = of_find_compatible_node(NULL, NULL, + "ti,clkctrl"); + if (np) { has_clkctrl_data = true; + of_node_put(np); } else { clkctrl_nodes_missing = true; @@ -520,3 +525,74 @@ struct clk *ti_clk_register(struct device *dev, struct clk_hw *hw, return clk; } + +/** + * ti_clk_register_omap_hw - register a clk_hw_omap to the clock framework + * @dev: device for this clock + * @hw: hardware clock handle + * @con: connection ID for this clock + * + * Registers a clk_hw_omap clock to the clock framewor, adds a clock alias + * for it, and adds the list to the available clk_hw_omap type clocks. + * Returns a handle to the registered clock if successful, ERR_PTR value + * in failure. + */ +struct clk *ti_clk_register_omap_hw(struct device *dev, struct clk_hw *hw, + const char *con) +{ + struct clk *clk; + struct clk_hw_omap *oclk; + + clk = ti_clk_register(dev, hw, con); + if (IS_ERR(clk)) + return clk; + + oclk = to_clk_hw_omap(hw); + + list_add(&oclk->node, &clk_hw_omap_clocks); + + return clk; +} + +/** + * omap2_clk_for_each - call function for each registered clk_hw_omap + * @fn: pointer to a callback function + * + * Call @fn for each registered clk_hw_omap, passing @hw to each + * function. @fn must return 0 for success or any other value for + * failure. If @fn returns non-zero, the iteration across clocks + * will stop and the non-zero return value will be passed to the + * caller of omap2_clk_for_each(). + */ +int omap2_clk_for_each(int (*fn)(struct clk_hw_omap *hw)) +{ + int ret; + struct clk_hw_omap *hw; + + list_for_each_entry(hw, &clk_hw_omap_clocks, node) { + ret = (*fn)(hw); + if (ret) + break; + } + + return ret; +} + +/** + * omap2_clk_is_hw_omap - check if the provided clk_hw is OMAP clock + * @hw: clk_hw to check if it is an omap clock or not + * + * Checks if the provided clk_hw is OMAP clock or not. Returns true if + * it is, false otherwise. + */ +bool omap2_clk_is_hw_omap(struct clk_hw *hw) +{ + struct clk_hw_omap *oclk; + + list_for_each_entry(oclk, &clk_hw_omap_clocks, node) { + if (&oclk->hw == hw) + return true; + } + + return false; +} diff --git a/drivers/clk/ti/clkctrl.c b/drivers/clk/ti/clkctrl.c index 40630eb950fcb..639f515e08f09 100644 --- a/drivers/clk/ti/clkctrl.c +++ b/drivers/clk/ti/clkctrl.c @@ -276,7 +276,7 @@ _ti_clkctrl_clk_register(struct omap_clkctrl_provider *provider, init.parent_names = parents; init.num_parents = num_parents; init.ops = ops; - init.flags = CLK_IS_BASIC; + init.flags = 0; clk = ti_clk_register(NULL, clk_hw, init.name); if (IS_ERR_OR_NULL(clk)) { @@ -530,7 +530,7 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node) * Create default clkdm name, replace _cm from end of parent * node name with _clkdm */ - provider->clkdm_name[strlen(provider->clkdm_name) - 5] = 0; + provider->clkdm_name[strlen(provider->clkdm_name) - 2] = 0; } else { provider->clkdm_name = kasprintf(GFP_KERNEL, "%pOFn", node); if (!provider->clkdm_name) { diff --git a/drivers/clk/ti/clock.h b/drivers/clk/ti/clock.h index 9f312a2195100..1c0fac59d8090 100644 --- a/drivers/clk/ti/clock.h +++ b/drivers/clk/ti/clock.h @@ -203,6 +203,8 @@ typedef void (*ti_of_clk_init_cb_t)(void *, struct device_node *); struct clk *ti_clk_register(struct device *dev, struct clk_hw *hw, const char *con); +struct clk *ti_clk_register_omap_hw(struct device *dev, struct clk_hw *hw, + const char *con); int ti_clk_add_alias(struct device *dev, struct clk *clk, const char *con); void ti_clk_add_aliases(void); @@ -221,7 +223,6 @@ int ti_clk_retry_init(struct device_node *node, void *user, ti_of_clk_init_cb_t func); int ti_clk_add_component(struct device_node *node, struct clk_hw *hw, int type); -void omap2_init_clk_hw_omap_clocks(struct clk_hw *hw); int of_ti_clk_autoidle_setup(struct device_node *node); void omap2_clk_enable_init_clocks(const char **clk_names, u8 num_clocks); @@ -301,6 +302,8 @@ long omap4_dpll_regm4xen_round_rate(struct clk_hw *hw, unsigned long *parent_rate); int omap4_dpll_regm4xen_determine_rate(struct clk_hw *hw, struct clk_rate_request *req); +int omap2_clk_for_each(int (*fn)(struct clk_hw_omap *hw)); +bool omap2_clk_is_hw_omap(struct clk_hw *hw); extern struct ti_clk_ll_ops *ti_clk_ll_ops; diff --git a/drivers/clk/ti/clockdomain.c b/drivers/clk/ti/clockdomain.c index 07a805125e98c..423a99b9f10c7 100644 --- a/drivers/clk/ti/clockdomain.c +++ b/drivers/clk/ti/clockdomain.c @@ -143,7 +143,7 @@ static void __init of_ti_clockdomain_setup(struct device_node *node) continue; } clk_hw = __clk_get_hw(clk); - if (clk_hw_get_flags(clk_hw) & CLK_IS_BASIC) { + if (!omap2_clk_is_hw_omap(clk_hw)) { pr_warn("can't setup clkdm for basic clk %s\n", __clk_get_name(clk)); continue; diff --git a/drivers/clk/ti/divider.c b/drivers/clk/ti/divider.c index 0241450f3eb3c..4786e0ebc2e8a 100644 --- a/drivers/clk/ti/divider.c +++ b/drivers/clk/ti/divider.c @@ -336,7 +336,7 @@ static struct clk *_register_divider(struct device *dev, const char *name, init.name = name; init.ops = &ti_clk_divider_ops; - init.flags = flags | CLK_IS_BASIC; + init.flags = flags; init.parent_names = (parent_name ? &parent_name : NULL); init.num_parents = (parent_name ? 1 : 0); diff --git a/drivers/clk/ti/dpll.c b/drivers/clk/ti/dpll.c index 6c3329bc116fb..659dadb23279c 100644 --- a/drivers/clk/ti/dpll.c +++ b/drivers/clk/ti/dpll.c @@ -192,10 +192,9 @@ static void __init _register_dpll(void *user, dd->clk_bypass = __clk_get_hw(clk); /* register the clock */ - clk = ti_clk_register(NULL, &clk_hw->hw, node->name); + clk = ti_clk_register_omap_hw(NULL, &clk_hw->hw, node->name); if (!IS_ERR(clk)) { - omap2_init_clk_hw_omap_clocks(&clk_hw->hw); of_clk_add_provider(node, of_clk_src_simple_get, clk); kfree(clk_hw->hw.init->parent_names); kfree(clk_hw->hw.init); @@ -265,14 +264,12 @@ static void _register_dpll_x2(struct device_node *node, #endif /* register the clock */ - clk = ti_clk_register(NULL, &clk_hw->hw, name); + clk = ti_clk_register_omap_hw(NULL, &clk_hw->hw, name); - if (IS_ERR(clk)) { + if (IS_ERR(clk)) kfree(clk_hw); - } else { - omap2_init_clk_hw_omap_clocks(&clk_hw->hw); + else of_clk_add_provider(node, of_clk_src_simple_get, clk); - } } #endif diff --git a/drivers/clk/ti/dpll3xxx.c b/drivers/clk/ti/dpll3xxx.c index 44b6b6403753c..3dde6c8c3354f 100644 --- a/drivers/clk/ti/dpll3xxx.c +++ b/drivers/clk/ti/dpll3xxx.c @@ -731,7 +731,7 @@ static struct clk_hw_omap *omap3_find_clkoutx2_dpll(struct clk_hw *hw) do { do { hw = clk_hw_get_parent(hw); - } while (hw && (clk_hw_get_flags(hw) & CLK_IS_BASIC)); + } while (hw && (!omap2_clk_is_hw_omap(hw))); if (!hw) break; pclk = to_clk_hw_omap(hw); diff --git a/drivers/clk/ti/gate.c b/drivers/clk/ti/gate.c index 1c78fff5513c7..504c0e91cdc76 100644 --- a/drivers/clk/ti/gate.c +++ b/drivers/clk/ti/gate.c @@ -123,7 +123,7 @@ static struct clk *_register_gate(struct device *dev, const char *name, init.flags = flags; - clk = ti_clk_register(NULL, &clk_hw->hw, name); + clk = ti_clk_register_omap_hw(NULL, &clk_hw->hw, name); if (IS_ERR(clk)) kfree(clk_hw); diff --git a/drivers/clk/ti/interface.c b/drivers/clk/ti/interface.c index 87e00c2ee957d..83e34429d3b10 100644 --- a/drivers/clk/ti/interface.c +++ b/drivers/clk/ti/interface.c @@ -57,12 +57,10 @@ static struct clk *_register_interface(struct device *dev, const char *name, init.num_parents = 1; init.parent_names = &parent_name; - clk = ti_clk_register(NULL, &clk_hw->hw, name); + clk = ti_clk_register_omap_hw(NULL, &clk_hw->hw, name); if (IS_ERR(clk)) kfree(clk_hw); - else - omap2_init_clk_hw_omap_clocks(&clk_hw->hw); return clk; } diff --git a/drivers/clk/ti/mux.c b/drivers/clk/ti/mux.c index 883bdde94d048..b7f9a4f068bf6 100644 --- a/drivers/clk/ti/mux.c +++ b/drivers/clk/ti/mux.c @@ -143,7 +143,7 @@ static struct clk *_register_mux(struct device *dev, const char *name, init.name = name; init.ops = &ti_clk_mux_ops; - init.flags = flags | CLK_IS_BASIC; + init.flags = flags; init.parent_names = parent_names; init.num_parents = num_parents; diff --git a/drivers/clk/uniphier/clk-uniphier-cpugear.c b/drivers/clk/uniphier/clk-uniphier-cpugear.c index ec11f55594ad0..5d2d42b7e182b 100644 --- a/drivers/clk/uniphier/clk-uniphier-cpugear.c +++ b/drivers/clk/uniphier/clk-uniphier-cpugear.c @@ -47,7 +47,7 @@ static int uniphier_clk_cpugear_set_parent(struct clk_hw *hw, u8 index) return ret; ret = regmap_write_bits(gear->regmap, - gear->regbase + UNIPHIER_CLK_CPUGEAR_SET, + gear->regbase + UNIPHIER_CLK_CPUGEAR_UPD, UNIPHIER_CLK_CPUGEAR_UPD_BIT, UNIPHIER_CLK_CPUGEAR_UPD_BIT); if (ret) diff --git a/drivers/clk/x86/clk-lpt.c b/drivers/clk/x86/clk-lpt.c index 6b40eb89ae190..68bd3abaef2c6 100644 --- a/drivers/clk/x86/clk-lpt.c +++ b/drivers/clk/x86/clk-lpt.c @@ -13,7 +13,7 @@ #include <linux/clk-provider.h> #include <linux/err.h> #include <linux/module.h> -#include <linux/platform_data/clk-lpss.h> +#include <linux/platform_data/x86/clk-lpss.h> #include <linux/platform_device.h> static int lpt_clk_probe(struct platform_device *pdev) diff --git a/drivers/clk/x86/clk-st.c b/drivers/clk/x86/clk-st.c index 3a0996f2d5564..25d4b97aff9ba 100644 --- a/drivers/clk/x86/clk-st.c +++ b/drivers/clk/x86/clk-st.c @@ -52,7 +52,8 @@ static int st_clk_probe(struct platform_device *pdev) 0, st_data->base + MISCCLKCNTL1, OSCCLKENB, CLK_GATE_SET_TO_DISABLE, NULL); - clk_hw_register_clkdev(hws[ST_CLK_GATE], "oscout1", NULL); + devm_clk_hw_register_clkdev(&pdev->dev, hws[ST_CLK_GATE], "oscout1", + NULL); return 0; } diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index a8b20b65bd4b7..aa4ec53281cea 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -1261,6 +1261,13 @@ static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void) return ARCH_TIMER_PHYS_SECURE_PPI; } +static void __init arch_timer_populate_kvm_info(void) +{ + arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI]; + if (is_kernel_in_hyp_mode()) + arch_timer_kvm_info.physical_irq = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]; +} + static int __init arch_timer_of_init(struct device_node *np) { int i, ret; @@ -1275,7 +1282,7 @@ static int __init arch_timer_of_init(struct device_node *np) for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++) arch_timer_ppi[i] = irq_of_parse_and_map(np, i); - arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI]; + arch_timer_populate_kvm_info(); rate = arch_timer_get_cntfrq(); arch_timer_of_configure_rate(rate, np); @@ -1605,7 +1612,7 @@ static int __init arch_timer_acpi_init(struct acpi_table_header *table) arch_timer_ppi[ARCH_TIMER_HYP_PPI] = acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI); - arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI]; + arch_timer_populate_kvm_info(); /* * When probing via ACPI, we have no mechanism to override the sysreg diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 0e626b00053b6..e10922709d139 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -206,17 +206,15 @@ unsigned int cpufreq_generic_get(unsigned int cpu) EXPORT_SYMBOL_GPL(cpufreq_generic_get); /** - * cpufreq_cpu_get: returns policy for a cpu and marks it busy. + * cpufreq_cpu_get - Return policy for a CPU and mark it as busy. + * @cpu: CPU to find the policy for. * - * @cpu: cpu to find policy for. + * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment + * the kobject reference counter of that policy. Return a valid policy on + * success or NULL on failure. * - * This returns policy for 'cpu', returns NULL if it doesn't exist. - * It also increments the kobject reference count to mark it busy and so would - * require a corresponding call to cpufreq_cpu_put() to decrement it back. - * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be - * freed as that depends on the kobj count. - * - * Return: A valid policy on success, otherwise NULL on failure. + * The policy returned by this function has to be released with the help of + * cpufreq_cpu_put() to balance its kobject reference counter properly. */ struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) { @@ -243,12 +241,8 @@ struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) EXPORT_SYMBOL_GPL(cpufreq_cpu_get); /** - * cpufreq_cpu_put: Decrements the usage count of a policy - * - * @policy: policy earlier returned by cpufreq_cpu_get(). - * - * This decrements the kobject reference count incremented earlier by calling - * cpufreq_cpu_get(). + * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy. + * @policy: cpufreq policy returned by cpufreq_cpu_get(). */ void cpufreq_cpu_put(struct cpufreq_policy *policy) { diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 002f5169d4ebc..e22f0dbaebb1d 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1762,7 +1762,7 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time, /* Start over if the CPU may have been idle. */ if (delta_ns > TICK_NSEC) { cpu->iowait_boost = ONE_EIGHTH_FP; - } else if (cpu->iowait_boost) { + } else if (cpu->iowait_boost >= ONE_EIGHTH_FP) { cpu->iowait_boost <<= 1; if (cpu->iowait_boost > int_tofp(1)) cpu->iowait_boost = int_tofp(1); diff --git a/drivers/cpufreq/pxa2xx-cpufreq.c b/drivers/cpufreq/pxa2xx-cpufreq.c index 46254e5839829..74e0e0c20c462 100644 --- a/drivers/cpufreq/pxa2xx-cpufreq.c +++ b/drivers/cpufreq/pxa2xx-cpufreq.c @@ -143,7 +143,7 @@ static int pxa_cpufreq_change_voltage(const struct pxa_freqs *pxa_freq) return ret; } -static void __init pxa_cpufreq_init_voltages(void) +static void pxa_cpufreq_init_voltages(void) { vcc_core = regulator_get(NULL, "vcc_core"); if (IS_ERR(vcc_core)) { @@ -159,7 +159,7 @@ static int pxa_cpufreq_change_voltage(const struct pxa_freqs *pxa_freq) return 0; } -static void __init pxa_cpufreq_init_voltages(void) { } +static void pxa_cpufreq_init_voltages(void) { } #endif static void find_freq_tables(struct cpufreq_frequency_table **freq_table, diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c index bb93e5cf6a4ae..9fddf828a76f2 100644 --- a/drivers/cpuidle/governor.c +++ b/drivers/cpuidle/governor.c @@ -89,6 +89,7 @@ int cpuidle_register_governor(struct cpuidle_governor *gov) mutex_lock(&cpuidle_lock); if (__cpuidle_find_governor(gov->name) == NULL) { ret = 0; + list_add_tail(&gov->governor_list, &cpuidle_governors); if (!cpuidle_curr_governor || !strncasecmp(param_governor, gov->name, CPUIDLE_NAME_LEN) || (cpuidle_curr_governor->rating < gov->rating && diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index 61316fc51548a..5951604e7d5c6 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -186,7 +186,7 @@ static unsigned int get_typical_interval(struct menu_device *data, unsigned int min, max, thresh, avg; uint64_t sum, variance; - thresh = UINT_MAX; /* Discard outliers above this value */ + thresh = INT_MAX; /* Discard outliers above this value */ again: diff --git a/drivers/dax/Kconfig b/drivers/dax/Kconfig index e0700bf4893a3..5ef624fe3934c 100644 --- a/drivers/dax/Kconfig +++ b/drivers/dax/Kconfig @@ -23,12 +23,38 @@ config DEV_DAX config DEV_DAX_PMEM tristate "PMEM DAX: direct access to persistent memory" depends on LIBNVDIMM && NVDIMM_DAX && DEV_DAX + depends on m # until we can kill DEV_DAX_PMEM_COMPAT default DEV_DAX help Support raw access to persistent memory. Note that this driver consumes memory ranges allocated and exported by the libnvdimm sub-system. - Say Y if unsure + Say M if unsure + +config DEV_DAX_KMEM + tristate "KMEM DAX: volatile-use of persistent memory" + default DEV_DAX + depends on DEV_DAX + depends on MEMORY_HOTPLUG # for add_memory() and friends + help + Support access to persistent memory as if it were RAM. This + allows easier use of persistent memory by unmodified + applications. + + To use this feature, a DAX device must be unbound from the + device_dax driver (PMEM DAX) and bound to this kmem driver + on each boot. + + Say N if unsure. + +config DEV_DAX_PMEM_COMPAT + tristate "PMEM DAX: support the deprecated /sys/class/dax interface" + depends on DEV_DAX_PMEM + default DEV_DAX_PMEM + help + Older versions of the libdaxctl library expect to find all + device-dax instances under /sys/class/dax. If libdaxctl in + your distribution is older than v58 say M, otherwise say N. endif diff --git a/drivers/dax/Makefile b/drivers/dax/Makefile index 574286fac87ce..81f7d54dadfb3 100644 --- a/drivers/dax/Makefile +++ b/drivers/dax/Makefile @@ -1,8 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DAX) += dax.o obj-$(CONFIG_DEV_DAX) += device_dax.o -obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o +obj-$(CONFIG_DEV_DAX_KMEM) += kmem.o dax-y := super.o -dax_pmem-y := pmem.o +dax-y += bus.o device_dax-y := device.o + +obj-y += pmem/ diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c new file mode 100644 index 0000000000000..2109cfe80219d --- /dev/null +++ b/drivers/dax/bus.c @@ -0,0 +1,503 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */ +#include <linux/memremap.h> +#include <linux/device.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/dax.h> +#include "dax-private.h" +#include "bus.h" + +static struct class *dax_class; + +static DEFINE_MUTEX(dax_bus_lock); + +#define DAX_NAME_LEN 30 +struct dax_id { + struct list_head list; + char dev_name[DAX_NAME_LEN]; +}; + +static int dax_bus_uevent(struct device *dev, struct kobj_uevent_env *env) +{ + /* + * We only ever expect to handle device-dax instances, i.e. the + * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero + */ + return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0); +} + +static struct dax_device_driver *to_dax_drv(struct device_driver *drv) +{ + return container_of(drv, struct dax_device_driver, drv); +} + +static struct dax_id *__dax_match_id(struct dax_device_driver *dax_drv, + const char *dev_name) +{ + struct dax_id *dax_id; + + lockdep_assert_held(&dax_bus_lock); + + list_for_each_entry(dax_id, &dax_drv->ids, list) + if (sysfs_streq(dax_id->dev_name, dev_name)) + return dax_id; + return NULL; +} + +static int dax_match_id(struct dax_device_driver *dax_drv, struct device *dev) +{ + int match; + + mutex_lock(&dax_bus_lock); + match = !!__dax_match_id(dax_drv, dev_name(dev)); + mutex_unlock(&dax_bus_lock); + + return match; +} + +enum id_action { + ID_REMOVE, + ID_ADD, +}; + +static ssize_t do_id_store(struct device_driver *drv, const char *buf, + size_t count, enum id_action action) +{ + struct dax_device_driver *dax_drv = to_dax_drv(drv); + unsigned int region_id, id; + char devname[DAX_NAME_LEN]; + struct dax_id *dax_id; + ssize_t rc = count; + int fields; + + fields = sscanf(buf, "dax%d.%d", ®ion_id, &id); + if (fields != 2) + return -EINVAL; + sprintf(devname, "dax%d.%d", region_id, id); + if (!sysfs_streq(buf, devname)) + return -EINVAL; + + mutex_lock(&dax_bus_lock); + dax_id = __dax_match_id(dax_drv, buf); + if (!dax_id) { + if (action == ID_ADD) { + dax_id = kzalloc(sizeof(*dax_id), GFP_KERNEL); + if (dax_id) { + strncpy(dax_id->dev_name, buf, DAX_NAME_LEN); + list_add(&dax_id->list, &dax_drv->ids); + } else + rc = -ENOMEM; + } else + /* nothing to remove */; + } else if (action == ID_REMOVE) { + list_del(&dax_id->list); + kfree(dax_id); + } else + /* dax_id already added */; + mutex_unlock(&dax_bus_lock); + + if (rc < 0) + return rc; + if (action == ID_ADD) + rc = driver_attach(drv); + if (rc) + return rc; + return count; +} + +static ssize_t new_id_store(struct device_driver *drv, const char *buf, + size_t count) +{ + return do_id_store(drv, buf, count, ID_ADD); +} +static DRIVER_ATTR_WO(new_id); + +static ssize_t remove_id_store(struct device_driver *drv, const char *buf, + size_t count) +{ + return do_id_store(drv, buf, count, ID_REMOVE); +} +static DRIVER_ATTR_WO(remove_id); + +static struct attribute *dax_drv_attrs[] = { + &driver_attr_new_id.attr, + &driver_attr_remove_id.attr, + NULL, +}; +ATTRIBUTE_GROUPS(dax_drv); + +static int dax_bus_match(struct device *dev, struct device_driver *drv); + +static struct bus_type dax_bus_type = { + .name = "dax", + .uevent = dax_bus_uevent, + .match = dax_bus_match, + .drv_groups = dax_drv_groups, +}; + +static int dax_bus_match(struct device *dev, struct device_driver *drv) +{ + struct dax_device_driver *dax_drv = to_dax_drv(drv); + + /* + * All but the 'device-dax' driver, which has 'match_always' + * set, requires an exact id match. + */ + if (dax_drv->match_always) + return 1; + + return dax_match_id(dax_drv, dev); +} + +/* + * Rely on the fact that drvdata is set before the attributes are + * registered, and that the attributes are unregistered before drvdata + * is cleared to assume that drvdata is always valid. + */ +static ssize_t id_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dax_region *dax_region = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", dax_region->id); +} +static DEVICE_ATTR_RO(id); + +static ssize_t region_size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dax_region *dax_region = dev_get_drvdata(dev); + + return sprintf(buf, "%llu\n", (unsigned long long) + resource_size(&dax_region->res)); +} +static struct device_attribute dev_attr_region_size = __ATTR(size, 0444, + region_size_show, NULL); + +static ssize_t align_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dax_region *dax_region = dev_get_drvdata(dev); + + return sprintf(buf, "%u\n", dax_region->align); +} +static DEVICE_ATTR_RO(align); + +static struct attribute *dax_region_attributes[] = { + &dev_attr_region_size.attr, + &dev_attr_align.attr, + &dev_attr_id.attr, + NULL, +}; + +static const struct attribute_group dax_region_attribute_group = { + .name = "dax_region", + .attrs = dax_region_attributes, +}; + +static const struct attribute_group *dax_region_attribute_groups[] = { + &dax_region_attribute_group, + NULL, +}; + +static void dax_region_free(struct kref *kref) +{ + struct dax_region *dax_region; + + dax_region = container_of(kref, struct dax_region, kref); + kfree(dax_region); +} + +void dax_region_put(struct dax_region *dax_region) +{ + kref_put(&dax_region->kref, dax_region_free); +} +EXPORT_SYMBOL_GPL(dax_region_put); + +static void dax_region_unregister(void *region) +{ + struct dax_region *dax_region = region; + + sysfs_remove_groups(&dax_region->dev->kobj, + dax_region_attribute_groups); + dax_region_put(dax_region); +} + +struct dax_region *alloc_dax_region(struct device *parent, int region_id, + struct resource *res, int target_node, unsigned int align, + unsigned long pfn_flags) +{ + struct dax_region *dax_region; + + /* + * The DAX core assumes that it can store its private data in + * parent->driver_data. This WARN is a reminder / safeguard for + * developers of device-dax drivers. + */ + if (dev_get_drvdata(parent)) { + dev_WARN(parent, "dax core failed to setup private data\n"); + return NULL; + } + + if (!IS_ALIGNED(res->start, align) + || !IS_ALIGNED(resource_size(res), align)) + return NULL; + + dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL); + if (!dax_region) + return NULL; + + dev_set_drvdata(parent, dax_region); + memcpy(&dax_region->res, res, sizeof(*res)); + dax_region->pfn_flags = pfn_flags; + kref_init(&dax_region->kref); + dax_region->id = region_id; + dax_region->align = align; + dax_region->dev = parent; + dax_region->target_node = target_node; + if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) { + kfree(dax_region); + return NULL; + } + + kref_get(&dax_region->kref); + if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region)) + return NULL; + return dax_region; +} +EXPORT_SYMBOL_GPL(alloc_dax_region); + +static ssize_t size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + unsigned long long size = resource_size(&dev_dax->region->res); + + return sprintf(buf, "%llu\n", size); +} +static DEVICE_ATTR_RO(size); + +static int dev_dax_target_node(struct dev_dax *dev_dax) +{ + struct dax_region *dax_region = dev_dax->region; + + return dax_region->target_node; +} + +static ssize_t target_node_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + + return sprintf(buf, "%d\n", dev_dax_target_node(dev_dax)); +} +static DEVICE_ATTR_RO(target_node); + +static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + /* + * We only ever expect to handle device-dax instances, i.e. the + * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero + */ + return sprintf(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0); +} +static DEVICE_ATTR_RO(modalias); + +static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct dev_dax *dev_dax = to_dev_dax(dev); + + if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0) + return 0; + return a->mode; +} + +static struct attribute *dev_dax_attributes[] = { + &dev_attr_modalias.attr, + &dev_attr_size.attr, + &dev_attr_target_node.attr, + NULL, +}; + +static const struct attribute_group dev_dax_attribute_group = { + .attrs = dev_dax_attributes, + .is_visible = dev_dax_visible, +}; + +static const struct attribute_group *dax_attribute_groups[] = { + &dev_dax_attribute_group, + NULL, +}; + +void kill_dev_dax(struct dev_dax *dev_dax) +{ + struct dax_device *dax_dev = dev_dax->dax_dev; + struct inode *inode = dax_inode(dax_dev); + + kill_dax(dax_dev); + unmap_mapping_range(inode->i_mapping, 0, 0, 1); +} +EXPORT_SYMBOL_GPL(kill_dev_dax); + +static void dev_dax_release(struct device *dev) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + struct dax_region *dax_region = dev_dax->region; + struct dax_device *dax_dev = dev_dax->dax_dev; + + dax_region_put(dax_region); + put_dax(dax_dev); + kfree(dev_dax); +} + +static void unregister_dev_dax(void *dev) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + + dev_dbg(dev, "%s\n", __func__); + + kill_dev_dax(dev_dax); + device_del(dev); + put_device(dev); +} + +struct dev_dax *__devm_create_dev_dax(struct dax_region *dax_region, int id, + struct dev_pagemap *pgmap, enum dev_dax_subsys subsys) +{ + struct device *parent = dax_region->dev; + struct dax_device *dax_dev; + struct dev_dax *dev_dax; + struct inode *inode; + struct device *dev; + int rc = -ENOMEM; + + if (id < 0) + return ERR_PTR(-EINVAL); + + dev_dax = kzalloc(sizeof(*dev_dax), GFP_KERNEL); + if (!dev_dax) + return ERR_PTR(-ENOMEM); + + memcpy(&dev_dax->pgmap, pgmap, sizeof(*pgmap)); + + /* + * No 'host' or dax_operations since there is no access to this + * device outside of mmap of the resulting character device. + */ + dax_dev = alloc_dax(dev_dax, NULL, NULL); + if (!dax_dev) + goto err; + + /* a device_dax instance is dead while the driver is not attached */ + kill_dax(dax_dev); + + /* from here on we're committed to teardown via dax_dev_release() */ + dev = &dev_dax->dev; + device_initialize(dev); + + dev_dax->dax_dev = dax_dev; + dev_dax->region = dax_region; + dev_dax->target_node = dax_region->target_node; + kref_get(&dax_region->kref); + + inode = dax_inode(dax_dev); + dev->devt = inode->i_rdev; + if (subsys == DEV_DAX_BUS) + dev->bus = &dax_bus_type; + else + dev->class = dax_class; + dev->parent = parent; + dev->groups = dax_attribute_groups; + dev->release = dev_dax_release; + dev_set_name(dev, "dax%d.%d", dax_region->id, id); + + rc = device_add(dev); + if (rc) { + kill_dev_dax(dev_dax); + put_device(dev); + return ERR_PTR(rc); + } + + rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev); + if (rc) + return ERR_PTR(rc); + + return dev_dax; + + err: + kfree(dev_dax); + + return ERR_PTR(rc); +} +EXPORT_SYMBOL_GPL(__devm_create_dev_dax); + +static int match_always_count; + +int __dax_driver_register(struct dax_device_driver *dax_drv, + struct module *module, const char *mod_name) +{ + struct device_driver *drv = &dax_drv->drv; + int rc = 0; + + INIT_LIST_HEAD(&dax_drv->ids); + drv->owner = module; + drv->name = mod_name; + drv->mod_name = mod_name; + drv->bus = &dax_bus_type; + + /* there can only be one default driver */ + mutex_lock(&dax_bus_lock); + match_always_count += dax_drv->match_always; + if (match_always_count > 1) { + match_always_count--; + WARN_ON(1); + rc = -EINVAL; + } + mutex_unlock(&dax_bus_lock); + if (rc) + return rc; + return driver_register(drv); +} +EXPORT_SYMBOL_GPL(__dax_driver_register); + +void dax_driver_unregister(struct dax_device_driver *dax_drv) +{ + struct device_driver *drv = &dax_drv->drv; + struct dax_id *dax_id, *_id; + + mutex_lock(&dax_bus_lock); + match_always_count -= dax_drv->match_always; + list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) { + list_del(&dax_id->list); + kfree(dax_id); + } + mutex_unlock(&dax_bus_lock); + driver_unregister(drv); +} +EXPORT_SYMBOL_GPL(dax_driver_unregister); + +int __init dax_bus_init(void) +{ + int rc; + + if (IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT)) { + dax_class = class_create(THIS_MODULE, "dax"); + if (IS_ERR(dax_class)) + return PTR_ERR(dax_class); + } + + rc = bus_register(&dax_bus_type); + if (rc) + class_destroy(dax_class); + return rc; +} + +void __exit dax_bus_exit(void) +{ + bus_unregister(&dax_bus_type); + class_destroy(dax_class); +} diff --git a/drivers/dax/bus.h b/drivers/dax/bus.h new file mode 100644 index 0000000000000..8619e32999436 --- /dev/null +++ b/drivers/dax/bus.h @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016 - 2018 Intel Corporation. All rights reserved. */ +#ifndef __DAX_BUS_H__ +#define __DAX_BUS_H__ +#include <linux/device.h> + +struct dev_dax; +struct resource; +struct dax_device; +struct dax_region; +void dax_region_put(struct dax_region *dax_region); +struct dax_region *alloc_dax_region(struct device *parent, int region_id, + struct resource *res, int target_node, unsigned int align, + unsigned long flags); + +enum dev_dax_subsys { + DEV_DAX_BUS, + DEV_DAX_CLASS, +}; + +struct dev_dax *__devm_create_dev_dax(struct dax_region *dax_region, int id, + struct dev_pagemap *pgmap, enum dev_dax_subsys subsys); + +static inline struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region, + int id, struct dev_pagemap *pgmap) +{ + return __devm_create_dev_dax(dax_region, id, pgmap, DEV_DAX_BUS); +} + +/* to be deleted when DEV_DAX_CLASS is removed */ +struct dev_dax *__dax_pmem_probe(struct device *dev, enum dev_dax_subsys subsys); + +struct dax_device_driver { + struct device_driver drv; + struct list_head ids; + int match_always; +}; + +int __dax_driver_register(struct dax_device_driver *dax_drv, + struct module *module, const char *mod_name); +#define dax_driver_register(driver) \ + __dax_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) +void dax_driver_unregister(struct dax_device_driver *dax_drv); +void kill_dev_dax(struct dev_dax *dev_dax); + +#if IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT) +int dev_dax_probe(struct device *dev); +#endif + +/* + * While run_dax() is potentially a generic operation that could be + * defined in include/linux/dax.h we don't want to grow any users + * outside of drivers/dax/ + */ +void run_dax(struct dax_device *dax_dev); + +#define MODULE_ALIAS_DAX_DEVICE(type) \ + MODULE_ALIAS("dax:t" __stringify(type) "*") +#define DAX_DEVICE_MODALIAS_FMT "dax:t%d" + +#endif /* __DAX_BUS_H__ */ diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h index b6fc4f04636de..a45612148ca02 100644 --- a/drivers/dax/dax-private.h +++ b/drivers/dax/dax-private.h @@ -16,10 +16,17 @@ #include <linux/device.h> #include <linux/cdev.h> +/* private routines between core files */ +struct dax_device; +struct dax_device *inode_dax(struct inode *inode); +struct inode *dax_inode(struct dax_device *dax_dev); +int dax_bus_init(void); +void dax_bus_exit(void); + /** * struct dax_region - mapping infrastructure for dax devices * @id: kernel-wide unique region for a memory range - * @base: linear address corresponding to @res + * @target_node: effective numa node if this memory range is onlined * @kref: to pin while other agents have a need to do lookups * @dev: parent device backing this region * @align: allocation and mapping alignment for child dax devices @@ -28,8 +35,7 @@ */ struct dax_region { int id; - struct ida ida; - void *base; + int target_node; struct kref kref; struct device *dev; unsigned int align; @@ -38,20 +44,28 @@ struct dax_region { }; /** - * struct dev_dax - instance data for a subdivision of a dax region + * struct dev_dax - instance data for a subdivision of a dax region, and + * data while the device is activated in the driver. * @region - parent region * @dax_dev - core dax functionality + * @target_node: effective numa node if dev_dax memory range is onlined * @dev - device core - * @id - child id in the region - * @num_resources - number of physical address extents in this device - * @res - array of physical address ranges + * @pgmap - pgmap for memmap setup / lifetime (driver owned) + * @ref: pgmap reference count (driver owned) + * @cmp: @ref final put completion (driver owned) */ struct dev_dax { struct dax_region *region; struct dax_device *dax_dev; + int target_node; struct device dev; - int id; - int num_resources; - struct resource res[0]; + struct dev_pagemap pgmap; + struct percpu_ref ref; + struct completion cmp; }; + +static inline struct dev_dax *to_dev_dax(struct device *dev) +{ + return container_of(dev, struct dev_dax, dev); +} #endif diff --git a/drivers/dax/dax.h b/drivers/dax/dax.h deleted file mode 100644 index f9e5feea742cd..0000000000000 --- a/drivers/dax/dax.h +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright(c) 2016 - 2017 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#ifndef __DAX_H__ -#define __DAX_H__ -struct dax_device; -struct dax_device *inode_dax(struct inode *inode); -struct inode *dax_inode(struct dax_device *dax_dev); -#endif /* __DAX_H__ */ diff --git a/drivers/dax/device-dax.h b/drivers/dax/device-dax.h deleted file mode 100644 index 688b051750bd7..0000000000000 --- a/drivers/dax/device-dax.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright(c) 2016 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#ifndef __DEVICE_DAX_H__ -#define __DEVICE_DAX_H__ -struct device; -struct dev_dax; -struct resource; -struct dax_region; -void dax_region_put(struct dax_region *dax_region); -struct dax_region *alloc_dax_region(struct device *parent, - int region_id, struct resource *res, unsigned int align, - void *addr, unsigned long flags); -struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region, - int id, struct resource *res, int count); -#endif /* __DEVICE_DAX_H__ */ diff --git a/drivers/dax/device.c b/drivers/dax/device.c index 948806e57cee3..e428468ab6618 100644 --- a/drivers/dax/device.c +++ b/drivers/dax/device.c @@ -1,15 +1,6 @@ -/* - * Copyright(c) 2016 - 2017 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */ +#include <linux/memremap.h> #include <linux/pagemap.h> #include <linux/module.h> #include <linux/device.h> @@ -21,161 +12,39 @@ #include <linux/mm.h> #include <linux/mman.h> #include "dax-private.h" -#include "dax.h" +#include "bus.h" -static struct class *dax_class; - -/* - * Rely on the fact that drvdata is set before the attributes are - * registered, and that the attributes are unregistered before drvdata - * is cleared to assume that drvdata is always valid. - */ -static ssize_t id_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct dax_region *dax_region = dev_get_drvdata(dev); - - return sprintf(buf, "%d\n", dax_region->id); -} -static DEVICE_ATTR_RO(id); - -static ssize_t region_size_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct dax_region *dax_region = dev_get_drvdata(dev); - - return sprintf(buf, "%llu\n", (unsigned long long) - resource_size(&dax_region->res)); -} -static struct device_attribute dev_attr_region_size = __ATTR(size, 0444, - region_size_show, NULL); - -static ssize_t align_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct dax_region *dax_region = dev_get_drvdata(dev); - - return sprintf(buf, "%u\n", dax_region->align); -} -static DEVICE_ATTR_RO(align); - -static struct attribute *dax_region_attributes[] = { - &dev_attr_region_size.attr, - &dev_attr_align.attr, - &dev_attr_id.attr, - NULL, -}; - -static const struct attribute_group dax_region_attribute_group = { - .name = "dax_region", - .attrs = dax_region_attributes, -}; - -static const struct attribute_group *dax_region_attribute_groups[] = { - &dax_region_attribute_group, - NULL, -}; - -static void dax_region_free(struct kref *kref) -{ - struct dax_region *dax_region; - - dax_region = container_of(kref, struct dax_region, kref); - kfree(dax_region); -} - -void dax_region_put(struct dax_region *dax_region) +static struct dev_dax *ref_to_dev_dax(struct percpu_ref *ref) { - kref_put(&dax_region->kref, dax_region_free); + return container_of(ref, struct dev_dax, ref); } -EXPORT_SYMBOL_GPL(dax_region_put); -static void dax_region_unregister(void *region) +static void dev_dax_percpu_release(struct percpu_ref *ref) { - struct dax_region *dax_region = region; + struct dev_dax *dev_dax = ref_to_dev_dax(ref); - sysfs_remove_groups(&dax_region->dev->kobj, - dax_region_attribute_groups); - dax_region_put(dax_region); + dev_dbg(&dev_dax->dev, "%s\n", __func__); + complete(&dev_dax->cmp); } -struct dax_region *alloc_dax_region(struct device *parent, int region_id, - struct resource *res, unsigned int align, void *addr, - unsigned long pfn_flags) +static void dev_dax_percpu_exit(void *data) { - struct dax_region *dax_region; - - /* - * The DAX core assumes that it can store its private data in - * parent->driver_data. This WARN is a reminder / safeguard for - * developers of device-dax drivers. - */ - if (dev_get_drvdata(parent)) { - dev_WARN(parent, "dax core failed to setup private data\n"); - return NULL; - } - - if (!IS_ALIGNED(res->start, align) - || !IS_ALIGNED(resource_size(res), align)) - return NULL; - - dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL); - if (!dax_region) - return NULL; - - dev_set_drvdata(parent, dax_region); - memcpy(&dax_region->res, res, sizeof(*res)); - dax_region->pfn_flags = pfn_flags; - kref_init(&dax_region->kref); - dax_region->id = region_id; - ida_init(&dax_region->ida); - dax_region->align = align; - dax_region->dev = parent; - dax_region->base = addr; - if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) { - kfree(dax_region); - return NULL; - } + struct percpu_ref *ref = data; + struct dev_dax *dev_dax = ref_to_dev_dax(ref); - kref_get(&dax_region->kref); - if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region)) - return NULL; - return dax_region; + dev_dbg(&dev_dax->dev, "%s\n", __func__); + wait_for_completion(&dev_dax->cmp); + percpu_ref_exit(ref); } -EXPORT_SYMBOL_GPL(alloc_dax_region); -static struct dev_dax *to_dev_dax(struct device *dev) +static void dev_dax_percpu_kill(struct percpu_ref *data) { - return container_of(dev, struct dev_dax, dev); -} - -static ssize_t size_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct dev_dax *dev_dax = to_dev_dax(dev); - unsigned long long size = 0; - int i; + struct percpu_ref *ref = data; + struct dev_dax *dev_dax = ref_to_dev_dax(ref); - for (i = 0; i < dev_dax->num_resources; i++) - size += resource_size(&dev_dax->res[i]); - - return sprintf(buf, "%llu\n", size); + dev_dbg(&dev_dax->dev, "%s\n", __func__); + percpu_ref_kill(ref); } -static DEVICE_ATTR_RO(size); - -static struct attribute *dev_dax_attributes[] = { - &dev_attr_size.attr, - NULL, -}; - -static const struct attribute_group dev_dax_attribute_group = { - .attrs = dev_dax_attributes, -}; - -static const struct attribute_group *dax_attribute_groups[] = { - &dev_dax_attribute_group, - NULL, -}; static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma, const char *func) @@ -226,21 +95,11 @@ static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma, __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, unsigned long size) { - struct resource *res; - /* gcc-4.6.3-nolibc for i386 complains that this is uninitialized */ - phys_addr_t uninitialized_var(phys); - int i; - - for (i = 0; i < dev_dax->num_resources; i++) { - res = &dev_dax->res[i]; - phys = pgoff * PAGE_SIZE + res->start; - if (phys >= res->start && phys <= res->end) - break; - pgoff -= PHYS_PFN(resource_size(res)); - } + struct resource *res = &dev_dax->region->res; + phys_addr_t phys; - if (i < dev_dax->num_resources) { - res = &dev_dax->res[i]; + phys = pgoff * PAGE_SIZE + res->start; + if (phys >= res->start && phys <= res->end) { if (phys + size - 1 <= res->end) return phys; } @@ -576,152 +435,100 @@ static const struct file_operations dax_fops = { .mmap_supported_flags = MAP_SYNC, }; -static void dev_dax_release(struct device *dev) +static void dev_dax_cdev_del(void *cdev) { - struct dev_dax *dev_dax = to_dev_dax(dev); - struct dax_region *dax_region = dev_dax->region; - struct dax_device *dax_dev = dev_dax->dax_dev; - - if (dev_dax->id >= 0) - ida_simple_remove(&dax_region->ida, dev_dax->id); - dax_region_put(dax_region); - put_dax(dax_dev); - kfree(dev_dax); + cdev_del(cdev); } -static void kill_dev_dax(struct dev_dax *dev_dax) +static void dev_dax_kill(void *dev_dax) { - struct dax_device *dax_dev = dev_dax->dax_dev; - struct inode *inode = dax_inode(dax_dev); - - kill_dax(dax_dev); - unmap_mapping_range(inode->i_mapping, 0, 0, 1); + kill_dev_dax(dev_dax); } -static void unregister_dev_dax(void *dev) +int dev_dax_probe(struct device *dev) { struct dev_dax *dev_dax = to_dev_dax(dev); struct dax_device *dax_dev = dev_dax->dax_dev; - struct inode *inode = dax_inode(dax_dev); - struct cdev *cdev = inode->i_cdev; - - dev_dbg(dev, "trace\n"); - - kill_dev_dax(dev_dax); - cdev_device_del(cdev, dev); - put_device(dev); -} - -struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region, - int id, struct resource *res, int count) -{ - struct device *parent = dax_region->dev; - struct dax_device *dax_dev; - struct dev_dax *dev_dax; + struct resource *res = &dev_dax->region->res; struct inode *inode; - struct device *dev; struct cdev *cdev; - int rc, i; - - if (!count) - return ERR_PTR(-EINVAL); - - dev_dax = kzalloc(struct_size(dev_dax, res, count), GFP_KERNEL); - if (!dev_dax) - return ERR_PTR(-ENOMEM); - - for (i = 0; i < count; i++) { - if (!IS_ALIGNED(res[i].start, dax_region->align) - || !IS_ALIGNED(resource_size(&res[i]), - dax_region->align)) { - rc = -EINVAL; - break; - } - dev_dax->res[i].start = res[i].start; - dev_dax->res[i].end = res[i].end; + void *addr; + int rc; + + /* 1:1 map region resource range to device-dax instance range */ + if (!devm_request_mem_region(dev, res->start, resource_size(res), + dev_name(dev))) { + dev_warn(dev, "could not reserve region %pR\n", res); + return -EBUSY; } - if (i < count) - goto err_id; + init_completion(&dev_dax->cmp); + rc = percpu_ref_init(&dev_dax->ref, dev_dax_percpu_release, 0, + GFP_KERNEL); + if (rc) + return rc; - if (id < 0) { - id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL); - dev_dax->id = id; - if (id < 0) { - rc = id; - goto err_id; - } - } else { - /* region provider owns @id lifetime */ - dev_dax->id = -1; - } + rc = devm_add_action_or_reset(dev, dev_dax_percpu_exit, &dev_dax->ref); + if (rc) + return rc; - /* - * No 'host' or dax_operations since there is no access to this - * device outside of mmap of the resulting character device. - */ - dax_dev = alloc_dax(dev_dax, NULL, NULL); - if (!dax_dev) { - rc = -ENOMEM; - goto err_dax; + dev_dax->pgmap.ref = &dev_dax->ref; + dev_dax->pgmap.kill = dev_dax_percpu_kill; + addr = devm_memremap_pages(dev, &dev_dax->pgmap); + if (IS_ERR(addr)) { + devm_remove_action(dev, dev_dax_percpu_exit, &dev_dax->ref); + percpu_ref_exit(&dev_dax->ref); + return PTR_ERR(addr); } - /* from here on we're committed to teardown via dax_dev_release() */ - dev = &dev_dax->dev; - device_initialize(dev); - inode = dax_inode(dax_dev); cdev = inode->i_cdev; cdev_init(cdev, &dax_fops); - cdev->owner = parent->driver->owner; - - dev_dax->num_resources = count; - dev_dax->dax_dev = dax_dev; - dev_dax->region = dax_region; - kref_get(&dax_region->kref); - - dev->devt = inode->i_rdev; - dev->class = dax_class; - dev->parent = parent; - dev->groups = dax_attribute_groups; - dev->release = dev_dax_release; - dev_set_name(dev, "dax%d.%d", dax_region->id, id); - - rc = cdev_device_add(cdev, dev); - if (rc) { - kill_dev_dax(dev_dax); - put_device(dev); - return ERR_PTR(rc); - } - - rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev); + if (dev->class) { + /* for the CONFIG_DEV_DAX_PMEM_COMPAT case */ + cdev->owner = dev->parent->driver->owner; + } else + cdev->owner = dev->driver->owner; + cdev_set_parent(cdev, &dev->kobj); + rc = cdev_add(cdev, dev->devt, 1); if (rc) - return ERR_PTR(rc); + return rc; - return dev_dax; + rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev); + if (rc) + return rc; - err_dax: - if (dev_dax->id >= 0) - ida_simple_remove(&dax_region->ida, dev_dax->id); - err_id: - kfree(dev_dax); + run_dax(dax_dev); + return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax); +} +EXPORT_SYMBOL_GPL(dev_dax_probe); - return ERR_PTR(rc); +static int dev_dax_remove(struct device *dev) +{ + /* all probe actions are unwound by devm */ + return 0; } -EXPORT_SYMBOL_GPL(devm_create_dev_dax); + +static struct dax_device_driver device_dax_driver = { + .drv = { + .probe = dev_dax_probe, + .remove = dev_dax_remove, + }, + .match_always = 1, +}; static int __init dax_init(void) { - dax_class = class_create(THIS_MODULE, "dax"); - return PTR_ERR_OR_ZERO(dax_class); + return dax_driver_register(&device_dax_driver); } static void __exit dax_exit(void) { - class_destroy(dax_class); + dax_driver_unregister(&device_dax_driver); } MODULE_AUTHOR("Intel Corporation"); MODULE_LICENSE("GPL v2"); -subsys_initcall(dax_init); +module_init(dax_init); module_exit(dax_exit); +MODULE_ALIAS_DAX_DEVICE(0); diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c new file mode 100644 index 0000000000000..a02318c6d28ab --- /dev/null +++ b/drivers/dax/kmem.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-2019 Intel Corporation. All rights reserved. */ +#include <linux/memremap.h> +#include <linux/pagemap.h> +#include <linux/memory.h> +#include <linux/module.h> +#include <linux/device.h> +#include <linux/pfn_t.h> +#include <linux/slab.h> +#include <linux/dax.h> +#include <linux/fs.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include "dax-private.h" +#include "bus.h" + +int dev_dax_kmem_probe(struct device *dev) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + struct resource *res = &dev_dax->region->res; + resource_size_t kmem_start; + resource_size_t kmem_size; + resource_size_t kmem_end; + struct resource *new_res; + int numa_node; + int rc; + + /* + * Ensure good NUMA information for the persistent memory. + * Without this check, there is a risk that slow memory + * could be mixed in a node with faster memory, causing + * unavoidable performance issues. + */ + numa_node = dev_dax->target_node; + if (numa_node < 0) { + dev_warn(dev, "rejecting DAX region %pR with invalid node: %d\n", + res, numa_node); + return -EINVAL; + } + + /* Hotplug starting at the beginning of the next block: */ + kmem_start = ALIGN(res->start, memory_block_size_bytes()); + + kmem_size = resource_size(res); + /* Adjust the size down to compensate for moving up kmem_start: */ + kmem_size -= kmem_start - res->start; + /* Align the size down to cover only complete blocks: */ + kmem_size &= ~(memory_block_size_bytes() - 1); + kmem_end = kmem_start + kmem_size; + + /* Region is permanently reserved. Hot-remove not yet implemented. */ + new_res = request_mem_region(kmem_start, kmem_size, dev_name(dev)); + if (!new_res) { + dev_warn(dev, "could not reserve region [%pa-%pa]\n", + &kmem_start, &kmem_end); + return -EBUSY; + } + + /* + * Set flags appropriate for System RAM. Leave ..._BUSY clear + * so that add_memory() can add a child resource. Do not + * inherit flags from the parent since it may set new flags + * unknown to us that will break add_memory() below. + */ + new_res->flags = IORESOURCE_SYSTEM_RAM; + new_res->name = dev_name(dev); + + rc = add_memory(numa_node, new_res->start, resource_size(new_res)); + if (rc) + return rc; + + return 0; +} + +static int dev_dax_kmem_remove(struct device *dev) +{ + /* + * Purposely leak the request_mem_region() for the device-dax + * range and return '0' to ->remove() attempts. The removal of + * the device from the driver always succeeds, but the region + * is permanently pinned as reserved by the unreleased + * request_mem_region(). + */ + return 0; +} + +static struct dax_device_driver device_dax_kmem_driver = { + .drv = { + .probe = dev_dax_kmem_probe, + .remove = dev_dax_kmem_remove, + }, +}; + +static int __init dax_kmem_init(void) +{ + return dax_driver_register(&device_dax_kmem_driver); +} + +static void __exit dax_kmem_exit(void) +{ + dax_driver_unregister(&device_dax_kmem_driver); +} + +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL v2"); +module_init(dax_kmem_init); +module_exit(dax_kmem_exit); +MODULE_ALIAS_DAX_DEVICE(0); diff --git a/drivers/dax/pmem.c b/drivers/dax/pmem.c deleted file mode 100644 index 2c1f459c0c63a..0000000000000 --- a/drivers/dax/pmem.c +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright(c) 2016 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include <linux/percpu-refcount.h> -#include <linux/memremap.h> -#include <linux/module.h> -#include <linux/pfn_t.h> -#include "../nvdimm/pfn.h" -#include "../nvdimm/nd.h" -#include "device-dax.h" - -struct dax_pmem { - struct device *dev; - struct percpu_ref ref; - struct dev_pagemap pgmap; - struct completion cmp; -}; - -static struct dax_pmem *to_dax_pmem(struct percpu_ref *ref) -{ - return container_of(ref, struct dax_pmem, ref); -} - -static void dax_pmem_percpu_release(struct percpu_ref *ref) -{ - struct dax_pmem *dax_pmem = to_dax_pmem(ref); - - dev_dbg(dax_pmem->dev, "trace\n"); - complete(&dax_pmem->cmp); -} - -static void dax_pmem_percpu_exit(void *data) -{ - struct percpu_ref *ref = data; - struct dax_pmem *dax_pmem = to_dax_pmem(ref); - - dev_dbg(dax_pmem->dev, "trace\n"); - wait_for_completion(&dax_pmem->cmp); - percpu_ref_exit(ref); -} - -static void dax_pmem_percpu_kill(struct percpu_ref *ref) -{ - struct dax_pmem *dax_pmem = to_dax_pmem(ref); - - dev_dbg(dax_pmem->dev, "trace\n"); - percpu_ref_kill(ref); -} - -static int dax_pmem_probe(struct device *dev) -{ - void *addr; - struct resource res; - int rc, id, region_id; - struct nd_pfn_sb *pfn_sb; - struct dev_dax *dev_dax; - struct dax_pmem *dax_pmem; - struct nd_namespace_io *nsio; - struct dax_region *dax_region; - struct nd_namespace_common *ndns; - struct nd_dax *nd_dax = to_nd_dax(dev); - struct nd_pfn *nd_pfn = &nd_dax->nd_pfn; - - ndns = nvdimm_namespace_common_probe(dev); - if (IS_ERR(ndns)) - return PTR_ERR(ndns); - nsio = to_nd_namespace_io(&ndns->dev); - - dax_pmem = devm_kzalloc(dev, sizeof(*dax_pmem), GFP_KERNEL); - if (!dax_pmem) - return -ENOMEM; - - /* parse the 'pfn' info block via ->rw_bytes */ - rc = devm_nsio_enable(dev, nsio); - if (rc) - return rc; - rc = nvdimm_setup_pfn(nd_pfn, &dax_pmem->pgmap); - if (rc) - return rc; - devm_nsio_disable(dev, nsio); - - pfn_sb = nd_pfn->pfn_sb; - - if (!devm_request_mem_region(dev, nsio->res.start, - resource_size(&nsio->res), - dev_name(&ndns->dev))) { - dev_warn(dev, "could not reserve region %pR\n", &nsio->res); - return -EBUSY; - } - - dax_pmem->dev = dev; - init_completion(&dax_pmem->cmp); - rc = percpu_ref_init(&dax_pmem->ref, dax_pmem_percpu_release, 0, - GFP_KERNEL); - if (rc) - return rc; - - rc = devm_add_action(dev, dax_pmem_percpu_exit, &dax_pmem->ref); - if (rc) { - percpu_ref_exit(&dax_pmem->ref); - return rc; - } - - dax_pmem->pgmap.ref = &dax_pmem->ref; - dax_pmem->pgmap.kill = dax_pmem_percpu_kill; - addr = devm_memremap_pages(dev, &dax_pmem->pgmap); - if (IS_ERR(addr)) - return PTR_ERR(addr); - - /* adjust the dax_region resource to the start of data */ - memcpy(&res, &dax_pmem->pgmap.res, sizeof(res)); - res.start += le64_to_cpu(pfn_sb->dataoff); - - rc = sscanf(dev_name(&ndns->dev), "namespace%d.%d", ®ion_id, &id); - if (rc != 2) - return -EINVAL; - - dax_region = alloc_dax_region(dev, region_id, &res, - le32_to_cpu(pfn_sb->align), addr, PFN_DEV|PFN_MAP); - if (!dax_region) - return -ENOMEM; - - /* TODO: support for subdividing a dax region... */ - dev_dax = devm_create_dev_dax(dax_region, id, &res, 1); - - /* child dev_dax instances now own the lifetime of the dax_region */ - dax_region_put(dax_region); - - return PTR_ERR_OR_ZERO(dev_dax); -} - -static struct nd_device_driver dax_pmem_driver = { - .probe = dax_pmem_probe, - .drv = { - .name = "dax_pmem", - }, - .type = ND_DRIVER_DAX_PMEM, -}; - -module_nd_driver(dax_pmem_driver); - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Intel Corporation"); -MODULE_ALIAS_ND_DEVICE(ND_DEVICE_DAX_PMEM); diff --git a/drivers/dax/pmem/Makefile b/drivers/dax/pmem/Makefile new file mode 100644 index 0000000000000..e2e79bd3fdcf9 --- /dev/null +++ b/drivers/dax/pmem/Makefile @@ -0,0 +1,7 @@ +obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o +obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem_core.o +obj-$(CONFIG_DEV_DAX_PMEM_COMPAT) += dax_pmem_compat.o + +dax_pmem-y := pmem.o +dax_pmem_core-y := core.o +dax_pmem_compat-y := compat.o diff --git a/drivers/dax/pmem/compat.c b/drivers/dax/pmem/compat.c new file mode 100644 index 0000000000000..d7b15e6f30c5b --- /dev/null +++ b/drivers/dax/pmem/compat.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016 - 2018 Intel Corporation. All rights reserved. */ +#include <linux/percpu-refcount.h> +#include <linux/memremap.h> +#include <linux/module.h> +#include <linux/pfn_t.h> +#include <linux/nd.h> +#include "../bus.h" + +/* we need the private definitions to implement compat suport */ +#include "../dax-private.h" + +static int dax_pmem_compat_probe(struct device *dev) +{ + struct dev_dax *dev_dax = __dax_pmem_probe(dev, DEV_DAX_CLASS); + int rc; + + if (IS_ERR(dev_dax)) + return PTR_ERR(dev_dax); + + if (!devres_open_group(&dev_dax->dev, dev_dax, GFP_KERNEL)) + return -ENOMEM; + + device_lock(&dev_dax->dev); + rc = dev_dax_probe(&dev_dax->dev); + device_unlock(&dev_dax->dev); + + devres_close_group(&dev_dax->dev, dev_dax); + if (rc) + devres_release_group(&dev_dax->dev, dev_dax); + + return rc; +} + +static int dax_pmem_compat_release(struct device *dev, void *data) +{ + device_lock(dev); + devres_release_group(dev, to_dev_dax(dev)); + device_unlock(dev); + + return 0; +} + +static int dax_pmem_compat_remove(struct device *dev) +{ + device_for_each_child(dev, NULL, dax_pmem_compat_release); + return 0; +} + +static struct nd_device_driver dax_pmem_compat_driver = { + .probe = dax_pmem_compat_probe, + .remove = dax_pmem_compat_remove, + .drv = { + .name = "dax_pmem_compat", + }, + .type = ND_DRIVER_DAX_PMEM, +}; + +static int __init dax_pmem_compat_init(void) +{ + return nd_driver_register(&dax_pmem_compat_driver); +} +module_init(dax_pmem_compat_init); + +static void __exit dax_pmem_compat_exit(void) +{ + driver_unregister(&dax_pmem_compat_driver.drv); +} +module_exit(dax_pmem_compat_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Intel Corporation"); +MODULE_ALIAS_ND_DEVICE(ND_DEVICE_DAX_PMEM); diff --git a/drivers/dax/pmem/core.c b/drivers/dax/pmem/core.c new file mode 100644 index 0000000000000..f71019ce06470 --- /dev/null +++ b/drivers/dax/pmem/core.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016 - 2018 Intel Corporation. All rights reserved. */ +#include <linux/memremap.h> +#include <linux/module.h> +#include <linux/pfn_t.h> +#include "../../nvdimm/pfn.h" +#include "../../nvdimm/nd.h" +#include "../bus.h" + +struct dev_dax *__dax_pmem_probe(struct device *dev, enum dev_dax_subsys subsys) +{ + struct resource res; + int rc, id, region_id; + resource_size_t offset; + struct nd_pfn_sb *pfn_sb; + struct dev_dax *dev_dax; + struct nd_namespace_io *nsio; + struct dax_region *dax_region; + struct dev_pagemap pgmap = { 0 }; + struct nd_namespace_common *ndns; + struct nd_dax *nd_dax = to_nd_dax(dev); + struct nd_pfn *nd_pfn = &nd_dax->nd_pfn; + struct nd_region *nd_region = to_nd_region(dev->parent); + + ndns = nvdimm_namespace_common_probe(dev); + if (IS_ERR(ndns)) + return ERR_CAST(ndns); + nsio = to_nd_namespace_io(&ndns->dev); + + /* parse the 'pfn' info block via ->rw_bytes */ + rc = devm_nsio_enable(dev, nsio); + if (rc) + return ERR_PTR(rc); + rc = nvdimm_setup_pfn(nd_pfn, &pgmap); + if (rc) + return ERR_PTR(rc); + devm_nsio_disable(dev, nsio); + + /* reserve the metadata area, device-dax will reserve the data */ + pfn_sb = nd_pfn->pfn_sb; + offset = le64_to_cpu(pfn_sb->dataoff); + if (!devm_request_mem_region(dev, nsio->res.start, offset, + dev_name(&ndns->dev))) { + dev_warn(dev, "could not reserve metadata\n"); + return ERR_PTR(-EBUSY); + } + + rc = sscanf(dev_name(&ndns->dev), "namespace%d.%d", ®ion_id, &id); + if (rc != 2) + return ERR_PTR(-EINVAL); + + /* adjust the dax_region resource to the start of data */ + memcpy(&res, &pgmap.res, sizeof(res)); + res.start += offset; + dax_region = alloc_dax_region(dev, region_id, &res, + nd_region->target_node, le32_to_cpu(pfn_sb->align), + PFN_DEV|PFN_MAP); + if (!dax_region) + return ERR_PTR(-ENOMEM); + + dev_dax = __devm_create_dev_dax(dax_region, id, &pgmap, subsys); + + /* child dev_dax instances now own the lifetime of the dax_region */ + dax_region_put(dax_region); + + return dev_dax; +} +EXPORT_SYMBOL_GPL(__dax_pmem_probe); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Intel Corporation"); diff --git a/drivers/dax/pmem/pmem.c b/drivers/dax/pmem/pmem.c new file mode 100644 index 0000000000000..0ae4238a0ef88 --- /dev/null +++ b/drivers/dax/pmem/pmem.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016 - 2018 Intel Corporation. All rights reserved. */ +#include <linux/percpu-refcount.h> +#include <linux/memremap.h> +#include <linux/module.h> +#include <linux/pfn_t.h> +#include <linux/nd.h> +#include "../bus.h" + +static int dax_pmem_probe(struct device *dev) +{ + return PTR_ERR_OR_ZERO(__dax_pmem_probe(dev, DEV_DAX_BUS)); +} + +static struct nd_device_driver dax_pmem_driver = { + .probe = dax_pmem_probe, + .drv = { + .name = "dax_pmem", + }, + .type = ND_DRIVER_DAX_PMEM, +}; + +static int __init dax_pmem_init(void) +{ + return nd_driver_register(&dax_pmem_driver); +} +module_init(dax_pmem_init); + +static void __exit dax_pmem_exit(void) +{ + driver_unregister(&dax_pmem_driver.drv); +} +module_exit(dax_pmem_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Intel Corporation"); +#if !IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT) +/* For compat builds, don't load this module by default */ +MODULE_ALIAS_ND_DEVICE(ND_DEVICE_DAX_PMEM); +#endif diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 0cb8c30ea2788..0a339b85133e1 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -22,6 +22,7 @@ #include <linux/uio.h> #include <linux/dax.h> #include <linux/fs.h> +#include "dax-private.h" static dev_t dax_devt; DEFINE_STATIC_SRCU(dax_srcu); @@ -383,11 +384,15 @@ void kill_dax(struct dax_device *dax_dev) spin_lock(&dax_host_lock); hlist_del_init(&dax_dev->list); spin_unlock(&dax_host_lock); - - dax_dev->private = NULL; } EXPORT_SYMBOL_GPL(kill_dax); +void run_dax(struct dax_device *dax_dev) +{ + set_bit(DAXDEV_ALIVE, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(run_dax); + static struct inode *dax_alloc_inode(struct super_block *sb) { struct dax_device *dax_dev; @@ -602,6 +607,8 @@ EXPORT_SYMBOL_GPL(dax_inode); void *dax_get_private(struct dax_device *dax_dev) { + if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags)) + return NULL; return dax_dev->private; } EXPORT_SYMBOL_GPL(dax_get_private); @@ -615,7 +622,7 @@ static void init_once(void *_dax_dev) inode_init_once(inode); } -static int __dax_fs_init(void) +static int dax_fs_init(void) { int rc; @@ -647,35 +654,45 @@ static int __dax_fs_init(void) return rc; } -static void __dax_fs_exit(void) +static void dax_fs_exit(void) { kern_unmount(dax_mnt); unregister_filesystem(&dax_fs_type); kmem_cache_destroy(dax_cache); } -static int __init dax_fs_init(void) +static int __init dax_core_init(void) { int rc; - rc = __dax_fs_init(); + rc = dax_fs_init(); if (rc) return rc; rc = alloc_chrdev_region(&dax_devt, 0, MINORMASK+1, "dax"); if (rc) - __dax_fs_exit(); - return rc; + goto err_chrdev; + + rc = dax_bus_init(); + if (rc) + goto err_bus; + return 0; + +err_bus: + unregister_chrdev_region(dax_devt, MINORMASK+1); +err_chrdev: + dax_fs_exit(); + return 0; } -static void __exit dax_fs_exit(void) +static void __exit dax_core_exit(void) { unregister_chrdev_region(dax_devt, MINORMASK+1); ida_destroy(&dax_minor_ida); - __dax_fs_exit(); + dax_fs_exit(); } MODULE_AUTHOR("Intel Corporation"); MODULE_LICENSE("GPL v2"); -subsys_initcall(dax_fs_init); -module_exit(dax_fs_exit); +subsys_initcall(dax_core_init); +module_exit(dax_core_exit); diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index d2286c7f7222b..0b1dfb5bf2d98 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -218,6 +218,20 @@ config FSL_EDMA multiplexing capability for DMA request sources(slot). This module can be found on Freescale Vybrid and LS-1 SoCs. +config FSL_QDMA + tristate "NXP Layerscape qDMA engine support" + depends on ARM || ARM64 + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + select DMA_ENGINE_RAID + select ASYNC_TX_ENABLE_CHANNEL_SWITCH + help + Support the NXP Layerscape qDMA engine with command queue and legacy mode. + Channel virtualization is supported through enqueuing of DMA jobs to, + or dequeuing DMA jobs from, different work queues. + This module can be found on NXP Layerscape SoCs. + The qdma driver only work on SoCs with a DPAA hardware block. + config FSL_RAID tristate "Freescale RAID engine Support" depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 09571a81353d6..6126e1c3a8756 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -33,6 +33,7 @@ obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o obj-$(CONFIG_FSL_DMA) += fsldma.o obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o obj-$(CONFIG_MCF_EDMA) += mcf-edma.o fsl-edma-common.o +obj-$(CONFIG_FSL_QDMA) += fsl-qdma.o obj-$(CONFIG_FSL_RAID) += fsl_raid.o obj-$(CONFIG_HSU_DMA) += hsu/ obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 01d936c9fe899..a0a9cd76c1d4d 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -134,7 +134,6 @@ static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) struct at_desc *ret = NULL; unsigned long flags; unsigned int i = 0; - LIST_HEAD(tmp_list); spin_lock_irqsave(&atchan->lock, flags); list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { @@ -1387,8 +1386,6 @@ static int atc_pause(struct dma_chan *chan) int chan_id = atchan->chan_common.chan_id; unsigned long flags; - LIST_HEAD(list); - dev_vdbg(chan2dev(chan), "%s\n", __func__); spin_lock_irqsave(&atchan->lock, flags); @@ -1408,8 +1405,6 @@ static int atc_resume(struct dma_chan *chan) int chan_id = atchan->chan_common.chan_id; unsigned long flags; - LIST_HEAD(list); - dev_vdbg(chan2dev(chan), "%s\n", __func__); if (!atc_chan_is_paused(atchan)) diff --git a/drivers/dma/bcm2835-dma.c b/drivers/dma/bcm2835-dma.c index ae10f5614f953..ec8a291d62bab 100644 --- a/drivers/dma/bcm2835-dma.c +++ b/drivers/dma/bcm2835-dma.c @@ -2,9 +2,6 @@ /* * BCM2835 DMA engine support * - * This driver only supports cyclic DMA transfers - * as needed for the I2S module. - * * Author: Florian Meier <florian.meier@koalo.de> * Copyright 2013 * @@ -42,7 +39,6 @@ struct bcm2835_dmadev { struct dma_device ddev; - spinlock_t lock; void __iomem *base; struct device_dma_parameters dma_parms; }; @@ -64,7 +60,6 @@ struct bcm2835_cb_entry { struct bcm2835_chan { struct virt_dma_chan vc; - struct list_head node; struct dma_slave_config cfg; unsigned int dreq; @@ -312,8 +307,7 @@ static struct bcm2835_desc *bcm2835_dma_create_cb_chain( return NULL; /* allocate and setup the descriptor. */ - d = kzalloc(sizeof(*d) + frames * sizeof(struct bcm2835_cb_entry), - gfp); + d = kzalloc(struct_size(d, cb_list, frames), gfp); if (!d) return NULL; @@ -406,7 +400,7 @@ static void bcm2835_dma_fill_cb_chain_with_sg( } } -static int bcm2835_dma_abort(struct bcm2835_chan *c) +static void bcm2835_dma_abort(struct bcm2835_chan *c) { void __iomem *chan_base = c->chan_base; long int timeout = 10000; @@ -416,7 +410,7 @@ static int bcm2835_dma_abort(struct bcm2835_chan *c) * (The ACTIVE flag in the CS register is not a reliable indicator.) */ if (!readl(chan_base + BCM2835_DMA_ADDR)) - return 0; + return; /* Write 0 to the active bit - Pause the DMA */ writel(0, chan_base + BCM2835_DMA_CS); @@ -432,7 +426,6 @@ static int bcm2835_dma_abort(struct bcm2835_chan *c) "failed to complete outstanding writes\n"); writel(BCM2835_DMA_RESET, chan_base + BCM2835_DMA_CS); - return 0; } static void bcm2835_dma_start_desc(struct bcm2835_chan *c) @@ -504,8 +497,12 @@ static int bcm2835_dma_alloc_chan_resources(struct dma_chan *chan) dev_dbg(dev, "Allocating DMA channel %d\n", c->ch); + /* + * Control blocks are 256 bit in length and must start at a 256 bit + * (32 byte) aligned address (BCM2835 ARM Peripherals, sec. 4.2.1.1). + */ c->cb_pool = dma_pool_create(dev_name(dev), dev, - sizeof(struct bcm2835_dma_cb), 0, 0); + sizeof(struct bcm2835_dma_cb), 32, 0); if (!c->cb_pool) { dev_err(dev, "unable to allocate descriptor pool\n"); return -ENOMEM; @@ -774,17 +771,11 @@ static int bcm2835_dma_slave_config(struct dma_chan *chan, static int bcm2835_dma_terminate_all(struct dma_chan *chan) { struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); - struct bcm2835_dmadev *d = to_bcm2835_dma_dev(c->vc.chan.device); unsigned long flags; LIST_HEAD(head); spin_lock_irqsave(&c->vc.lock, flags); - /* Prevent this channel being scheduled */ - spin_lock(&d->lock); - list_del_init(&c->node); - spin_unlock(&d->lock); - /* stop DMA activity */ if (c->desc) { vchan_terminate_vdesc(&c->desc->vd); @@ -817,7 +808,6 @@ static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id, c->vc.desc_free = bcm2835_dma_desc_free; vchan_init(&c->vc, &d->ddev); - INIT_LIST_HEAD(&c->node); c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id); c->ch = chan_id; @@ -920,7 +910,6 @@ static int bcm2835_dma_probe(struct platform_device *pdev) od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; od->ddev.dev = &pdev->dev; INIT_LIST_HEAD(&od->ddev.channels); - spin_lock_init(&od->lock); platform_set_drvdata(pdev, od); diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 15b2453d2647f..ffc0adc2f6ce4 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -367,8 +367,7 @@ static struct axi_dmac_desc *axi_dmac_alloc_desc(unsigned int num_sgs) struct axi_dmac_desc *desc; unsigned int i; - desc = kzalloc(sizeof(struct axi_dmac_desc) + - sizeof(struct axi_dmac_sg) * num_sgs, GFP_NOWAIT); + desc = kzalloc(struct_size(desc, sg, num_sgs), GFP_NOWAIT); if (!desc) return NULL; diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c index a8b6225faa12a..9ce0a386225b3 100644 --- a/drivers/dma/dma-jz4780.c +++ b/drivers/dma/dma-jz4780.c @@ -838,9 +838,8 @@ static int jz4780_dma_probe(struct platform_device *pdev) if (!soc_data) return -EINVAL; - jzdma = devm_kzalloc(dev, sizeof(*jzdma) - + sizeof(*jzdma->chan) * soc_data->nb_channels, - GFP_KERNEL); + jzdma = devm_kzalloc(dev, struct_size(jzdma, chan, + soc_data->nb_channels), GFP_KERNEL); if (!jzdma) return -ENOMEM; diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index 6511928b4cdfe..b96814a7dceb1 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -200,15 +200,20 @@ struct dmatest_done { wait_queue_head_t *wait; }; +struct dmatest_data { + u8 **raw; + u8 **aligned; + unsigned int cnt; + unsigned int off; +}; + struct dmatest_thread { struct list_head node; struct dmatest_info *info; struct task_struct *task; struct dma_chan *chan; - u8 **srcs; - u8 **usrcs; - u8 **dsts; - u8 **udsts; + struct dmatest_data src; + struct dmatest_data dst; enum dma_transaction_type type; wait_queue_head_t done_wait; struct dmatest_done test_done; @@ -481,6 +486,53 @@ static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len) return FIXPT_TO_INT(dmatest_persec(runtime, len >> 10)); } +static void __dmatest_free_test_data(struct dmatest_data *d, unsigned int cnt) +{ + unsigned int i; + + for (i = 0; i < cnt; i++) + kfree(d->raw[i]); + + kfree(d->aligned); + kfree(d->raw); +} + +static void dmatest_free_test_data(struct dmatest_data *d) +{ + __dmatest_free_test_data(d, d->cnt); +} + +static int dmatest_alloc_test_data(struct dmatest_data *d, + unsigned int buf_size, u8 align) +{ + unsigned int i = 0; + + d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL); + if (!d->raw) + return -ENOMEM; + + d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL); + if (!d->aligned) + goto err; + + for (i = 0; i < d->cnt; i++) { + d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL); + if (!d->raw[i]) + goto err; + + /* align to alignment restriction */ + if (align) + d->aligned[i] = PTR_ALIGN(d->raw[i], align); + else + d->aligned[i] = d->raw[i]; + } + + return 0; +err: + __dmatest_free_test_data(d, i); + return -ENOMEM; +} + /* * This function repeatedly tests DMA transfers of various lengths and * offsets for a given operation type until it is told to exit by @@ -511,8 +563,9 @@ static int dmatest_func(void *data) enum dma_ctrl_flags flags; u8 *pq_coefs = NULL; int ret; - int src_cnt; - int dst_cnt; + unsigned int buf_size; + struct dmatest_data *src; + struct dmatest_data *dst; int i; ktime_t ktime, start, diff; ktime_t filltime = 0; @@ -535,25 +588,27 @@ static int dmatest_func(void *data) params = &info->params; chan = thread->chan; dev = chan->device; + src = &thread->src; + dst = &thread->dst; if (thread->type == DMA_MEMCPY) { align = params->alignment < 0 ? dev->copy_align : params->alignment; - src_cnt = dst_cnt = 1; + src->cnt = dst->cnt = 1; } else if (thread->type == DMA_MEMSET) { align = params->alignment < 0 ? dev->fill_align : params->alignment; - src_cnt = dst_cnt = 1; + src->cnt = dst->cnt = 1; is_memset = true; } else if (thread->type == DMA_XOR) { /* force odd to ensure dst = src */ - src_cnt = min_odd(params->xor_sources | 1, dev->max_xor); - dst_cnt = 1; + src->cnt = min_odd(params->xor_sources | 1, dev->max_xor); + dst->cnt = 1; align = params->alignment < 0 ? dev->xor_align : params->alignment; } else if (thread->type == DMA_PQ) { /* force odd to ensure dst = src */ - src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); - dst_cnt = 2; + src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); + dst->cnt = 2; align = params->alignment < 0 ? dev->pq_align : params->alignment; @@ -561,75 +616,38 @@ static int dmatest_func(void *data) if (!pq_coefs) goto err_thread_type; - for (i = 0; i < src_cnt; i++) + for (i = 0; i < src->cnt; i++) pq_coefs[i] = 1; } else goto err_thread_type; /* Check if buffer count fits into map count variable (u8) */ - if ((src_cnt + dst_cnt) >= 255) { + if ((src->cnt + dst->cnt) >= 255) { pr_err("too many buffers (%d of 255 supported)\n", - src_cnt + dst_cnt); + src->cnt + dst->cnt); goto err_free_coefs; } - if (1 << align > params->buf_size) { + buf_size = params->buf_size; + if (1 << align > buf_size) { pr_err("%u-byte buffer too small for %d-byte alignment\n", - params->buf_size, 1 << align); + buf_size, 1 << align); goto err_free_coefs; } - thread->srcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL); - if (!thread->srcs) + if (dmatest_alloc_test_data(src, buf_size, align) < 0) goto err_free_coefs; - thread->usrcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL); - if (!thread->usrcs) - goto err_usrcs; - - for (i = 0; i < src_cnt; i++) { - thread->usrcs[i] = kmalloc(params->buf_size + align, - GFP_KERNEL); - if (!thread->usrcs[i]) - goto err_srcbuf; - - /* align srcs to alignment restriction */ - if (align) - thread->srcs[i] = PTR_ALIGN(thread->usrcs[i], align); - else - thread->srcs[i] = thread->usrcs[i]; - } - thread->srcs[i] = NULL; - - thread->dsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL); - if (!thread->dsts) - goto err_dsts; - - thread->udsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL); - if (!thread->udsts) - goto err_udsts; - - for (i = 0; i < dst_cnt; i++) { - thread->udsts[i] = kmalloc(params->buf_size + align, - GFP_KERNEL); - if (!thread->udsts[i]) - goto err_dstbuf; - - /* align dsts to alignment restriction */ - if (align) - thread->dsts[i] = PTR_ALIGN(thread->udsts[i], align); - else - thread->dsts[i] = thread->udsts[i]; - } - thread->dsts[i] = NULL; + if (dmatest_alloc_test_data(dst, buf_size, align) < 0) + goto err_src; set_user_nice(current, 10); - srcs = kcalloc(src_cnt, sizeof(dma_addr_t), GFP_KERNEL); + srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL); if (!srcs) - goto err_dstbuf; + goto err_dst; - dma_pq = kcalloc(dst_cnt, sizeof(dma_addr_t), GFP_KERNEL); + dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL); if (!dma_pq) goto err_srcs_array; @@ -644,21 +662,21 @@ static int dmatest_func(void *data) struct dma_async_tx_descriptor *tx = NULL; struct dmaengine_unmap_data *um; dma_addr_t *dsts; - unsigned int src_off, dst_off, len; + unsigned int len; total_tests++; if (params->transfer_size) { - if (params->transfer_size >= params->buf_size) { + if (params->transfer_size >= buf_size) { pr_err("%u-byte transfer size must be lower than %u-buffer size\n", - params->transfer_size, params->buf_size); + params->transfer_size, buf_size); break; } len = params->transfer_size; } else if (params->norandom) { - len = params->buf_size; + len = buf_size; } else { - len = dmatest_random() % params->buf_size + 1; + len = dmatest_random() % buf_size + 1; } /* Do not alter transfer size explicitly defined by user */ @@ -670,57 +688,57 @@ static int dmatest_func(void *data) total_len += len; if (params->norandom) { - src_off = 0; - dst_off = 0; + src->off = 0; + dst->off = 0; } else { - src_off = dmatest_random() % (params->buf_size - len + 1); - dst_off = dmatest_random() % (params->buf_size - len + 1); + src->off = dmatest_random() % (buf_size - len + 1); + dst->off = dmatest_random() % (buf_size - len + 1); - src_off = (src_off >> align) << align; - dst_off = (dst_off >> align) << align; + src->off = (src->off >> align) << align; + dst->off = (dst->off >> align) << align; } if (!params->noverify) { start = ktime_get(); - dmatest_init_srcs(thread->srcs, src_off, len, - params->buf_size, is_memset); - dmatest_init_dsts(thread->dsts, dst_off, len, - params->buf_size, is_memset); + dmatest_init_srcs(src->aligned, src->off, len, + buf_size, is_memset); + dmatest_init_dsts(dst->aligned, dst->off, len, + buf_size, is_memset); diff = ktime_sub(ktime_get(), start); filltime = ktime_add(filltime, diff); } - um = dmaengine_get_unmap_data(dev->dev, src_cnt + dst_cnt, + um = dmaengine_get_unmap_data(dev->dev, src->cnt + dst->cnt, GFP_KERNEL); if (!um) { failed_tests++; result("unmap data NULL", total_tests, - src_off, dst_off, len, ret); + src->off, dst->off, len, ret); continue; } - um->len = params->buf_size; - for (i = 0; i < src_cnt; i++) { - void *buf = thread->srcs[i]; + um->len = buf_size; + for (i = 0; i < src->cnt; i++) { + void *buf = src->aligned[i]; struct page *pg = virt_to_page(buf); unsigned long pg_off = offset_in_page(buf); um->addr[i] = dma_map_page(dev->dev, pg, pg_off, um->len, DMA_TO_DEVICE); - srcs[i] = um->addr[i] + src_off; + srcs[i] = um->addr[i] + src->off; ret = dma_mapping_error(dev->dev, um->addr[i]); if (ret) { result("src mapping error", total_tests, - src_off, dst_off, len, ret); + src->off, dst->off, len, ret); goto error_unmap_continue; } um->to_cnt++; } /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ - dsts = &um->addr[src_cnt]; - for (i = 0; i < dst_cnt; i++) { - void *buf = thread->dsts[i]; + dsts = &um->addr[src->cnt]; + for (i = 0; i < dst->cnt; i++) { + void *buf = dst->aligned[i]; struct page *pg = virt_to_page(buf); unsigned long pg_off = offset_in_page(buf); @@ -729,7 +747,7 @@ static int dmatest_func(void *data) ret = dma_mapping_error(dev->dev, dsts[i]); if (ret) { result("dst mapping error", total_tests, - src_off, dst_off, len, ret); + src->off, dst->off, len, ret); goto error_unmap_continue; } um->bidi_cnt++; @@ -737,29 +755,29 @@ static int dmatest_func(void *data) if (thread->type == DMA_MEMCPY) tx = dev->device_prep_dma_memcpy(chan, - dsts[0] + dst_off, + dsts[0] + dst->off, srcs[0], len, flags); else if (thread->type == DMA_MEMSET) tx = dev->device_prep_dma_memset(chan, - dsts[0] + dst_off, - *(thread->srcs[0] + src_off), + dsts[0] + dst->off, + *(src->aligned[0] + src->off), len, flags); else if (thread->type == DMA_XOR) tx = dev->device_prep_dma_xor(chan, - dsts[0] + dst_off, - srcs, src_cnt, + dsts[0] + dst->off, + srcs, src->cnt, len, flags); else if (thread->type == DMA_PQ) { - for (i = 0; i < dst_cnt; i++) - dma_pq[i] = dsts[i] + dst_off; + for (i = 0; i < dst->cnt; i++) + dma_pq[i] = dsts[i] + dst->off; tx = dev->device_prep_dma_pq(chan, dma_pq, srcs, - src_cnt, pq_coefs, + src->cnt, pq_coefs, len, flags); } if (!tx) { - result("prep error", total_tests, src_off, - dst_off, len, ret); + result("prep error", total_tests, src->off, + dst->off, len, ret); msleep(100); goto error_unmap_continue; } @@ -770,8 +788,8 @@ static int dmatest_func(void *data) cookie = tx->tx_submit(tx); if (dma_submit_error(cookie)) { - result("submit error", total_tests, src_off, - dst_off, len, ret); + result("submit error", total_tests, src->off, + dst->off, len, ret); msleep(100); goto error_unmap_continue; } @@ -783,58 +801,58 @@ static int dmatest_func(void *data) status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); if (!done->done) { - result("test timed out", total_tests, src_off, dst_off, + result("test timed out", total_tests, src->off, dst->off, len, 0); goto error_unmap_continue; } else if (status != DMA_COMPLETE) { result(status == DMA_ERROR ? "completion error status" : - "completion busy status", total_tests, src_off, - dst_off, len, ret); + "completion busy status", total_tests, src->off, + dst->off, len, ret); goto error_unmap_continue; } dmaengine_unmap_put(um); if (params->noverify) { - verbose_result("test passed", total_tests, src_off, - dst_off, len, 0); + verbose_result("test passed", total_tests, src->off, + dst->off, len, 0); continue; } start = ktime_get(); pr_debug("%s: verifying source buffer...\n", current->comm); - error_count = dmatest_verify(thread->srcs, 0, src_off, + error_count = dmatest_verify(src->aligned, 0, src->off, 0, PATTERN_SRC, true, is_memset); - error_count += dmatest_verify(thread->srcs, src_off, - src_off + len, src_off, + error_count += dmatest_verify(src->aligned, src->off, + src->off + len, src->off, PATTERN_SRC | PATTERN_COPY, true, is_memset); - error_count += dmatest_verify(thread->srcs, src_off + len, - params->buf_size, src_off + len, + error_count += dmatest_verify(src->aligned, src->off + len, + buf_size, src->off + len, PATTERN_SRC, true, is_memset); pr_debug("%s: verifying dest buffer...\n", current->comm); - error_count += dmatest_verify(thread->dsts, 0, dst_off, + error_count += dmatest_verify(dst->aligned, 0, dst->off, 0, PATTERN_DST, false, is_memset); - error_count += dmatest_verify(thread->dsts, dst_off, - dst_off + len, src_off, + error_count += dmatest_verify(dst->aligned, dst->off, + dst->off + len, src->off, PATTERN_SRC | PATTERN_COPY, false, is_memset); - error_count += dmatest_verify(thread->dsts, dst_off + len, - params->buf_size, dst_off + len, + error_count += dmatest_verify(dst->aligned, dst->off + len, + buf_size, dst->off + len, PATTERN_DST, false, is_memset); diff = ktime_sub(ktime_get(), start); comparetime = ktime_add(comparetime, diff); if (error_count) { - result("data error", total_tests, src_off, dst_off, + result("data error", total_tests, src->off, dst->off, len, error_count); failed_tests++; } else { - verbose_result("test passed", total_tests, src_off, - dst_off, len, 0); + verbose_result("test passed", total_tests, src->off, + dst->off, len, 0); } continue; @@ -852,19 +870,10 @@ error_unmap_continue: kfree(dma_pq); err_srcs_array: kfree(srcs); -err_dstbuf: - for (i = 0; thread->udsts[i]; i++) - kfree(thread->udsts[i]); - kfree(thread->udsts); -err_udsts: - kfree(thread->dsts); -err_dsts: -err_srcbuf: - for (i = 0; thread->usrcs[i]; i++) - kfree(thread->usrcs[i]); - kfree(thread->usrcs); -err_usrcs: - kfree(thread->srcs); +err_dst: + dmatest_free_test_data(dst); +err_src: + dmatest_free_test_data(src); err_free_coefs: kfree(pq_coefs); err_thread_type: diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index f8888dc0b8dc7..18b6014cf9b40 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -75,7 +75,7 @@ struct __packed axi_dma_lli { __le32 sstat; __le32 dstat; __le32 status_lo; - __le32 ststus_hi; + __le32 status_hi; __le32 reserved_lo; __le32 reserved_hi; }; diff --git a/drivers/dma/dw/Kconfig b/drivers/dma/dw/Kconfig index 04b9728c1d269..e5162690de8f1 100644 --- a/drivers/dma/dw/Kconfig +++ b/drivers/dma/dw/Kconfig @@ -1,3 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + # # DMA engine configuration for dw # diff --git a/drivers/dma/dw/Makefile b/drivers/dma/dw/Makefile index 2b949c2e45045..63ed895c09aaf 100644 --- a/drivers/dma/dw/Makefile +++ b/drivers/dma/dw/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DW_DMAC_CORE) += dw_dmac_core.o -dw_dmac_core-objs := core.o +dw_dmac_core-objs := core.o dw.o idma32.o obj-$(CONFIG_DW_DMAC) += dw_dmac.o dw_dmac-objs := platform.o diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index dc053e62f8945..21cb2a58dbd29 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Core driver for the Synopsys DesignWare DMA Controller * * Copyright (C) 2007-2008 Atmel Corporation * Copyright (C) 2010-2011 ST Microelectronics * Copyright (C) 2013 Intel Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include <linux/bitops.h> @@ -37,27 +34,6 @@ * support descriptor writeback. */ -#define DWC_DEFAULT_CTLLO(_chan) ({ \ - struct dw_dma_chan *_dwc = to_dw_dma_chan(_chan); \ - struct dma_slave_config *_sconfig = &_dwc->dma_sconfig; \ - bool _is_slave = is_slave_direction(_dwc->direction); \ - u8 _smsize = _is_slave ? _sconfig->src_maxburst : \ - DW_DMA_MSIZE_16; \ - u8 _dmsize = _is_slave ? _sconfig->dst_maxburst : \ - DW_DMA_MSIZE_16; \ - u8 _dms = (_dwc->direction == DMA_MEM_TO_DEV) ? \ - _dwc->dws.p_master : _dwc->dws.m_master; \ - u8 _sms = (_dwc->direction == DMA_DEV_TO_MEM) ? \ - _dwc->dws.p_master : _dwc->dws.m_master; \ - \ - (DWC_CTLL_DST_MSIZE(_dmsize) \ - | DWC_CTLL_SRC_MSIZE(_smsize) \ - | DWC_CTLL_LLP_D_EN \ - | DWC_CTLL_LLP_S_EN \ - | DWC_CTLL_DMS(_dms) \ - | DWC_CTLL_SMS(_sms)); \ - }) - /* The set of bus widths supported by the DMA controller */ #define DW_DMA_BUSWIDTHS \ BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ @@ -138,44 +114,6 @@ static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc) dwc->descs_allocated--; } -static void dwc_initialize_chan_idma32(struct dw_dma_chan *dwc) -{ - u32 cfghi = 0; - u32 cfglo = 0; - - /* Set default burst alignment */ - cfglo |= IDMA32C_CFGL_DST_BURST_ALIGN | IDMA32C_CFGL_SRC_BURST_ALIGN; - - /* Low 4 bits of the request lines */ - cfghi |= IDMA32C_CFGH_DST_PER(dwc->dws.dst_id & 0xf); - cfghi |= IDMA32C_CFGH_SRC_PER(dwc->dws.src_id & 0xf); - - /* Request line extension (2 bits) */ - cfghi |= IDMA32C_CFGH_DST_PER_EXT(dwc->dws.dst_id >> 4 & 0x3); - cfghi |= IDMA32C_CFGH_SRC_PER_EXT(dwc->dws.src_id >> 4 & 0x3); - - channel_writel(dwc, CFG_LO, cfglo); - channel_writel(dwc, CFG_HI, cfghi); -} - -static void dwc_initialize_chan_dw(struct dw_dma_chan *dwc) -{ - struct dw_dma *dw = to_dw_dma(dwc->chan.device); - u32 cfghi = DWC_CFGH_FIFO_MODE; - u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority); - bool hs_polarity = dwc->dws.hs_polarity; - - cfghi |= DWC_CFGH_DST_PER(dwc->dws.dst_id); - cfghi |= DWC_CFGH_SRC_PER(dwc->dws.src_id); - cfghi |= DWC_CFGH_PROTCTL(dw->pdata->protctl); - - /* Set polarity of handshake interface */ - cfglo |= hs_polarity ? DWC_CFGL_HS_DST_POL | DWC_CFGL_HS_SRC_POL : 0; - - channel_writel(dwc, CFG_LO, cfglo); - channel_writel(dwc, CFG_HI, cfghi); -} - static void dwc_initialize(struct dw_dma_chan *dwc) { struct dw_dma *dw = to_dw_dma(dwc->chan.device); @@ -183,10 +121,7 @@ static void dwc_initialize(struct dw_dma_chan *dwc) if (test_bit(DW_DMA_IS_INITIALIZED, &dwc->flags)) return; - if (dw->pdata->is_idma32) - dwc_initialize_chan_idma32(dwc); - else - dwc_initialize_chan_dw(dwc); + dw->initialize_chan(dwc); /* Enable interrupts */ channel_set_bit(dw, MASK.XFER, dwc->mask); @@ -215,37 +150,6 @@ static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc) cpu_relax(); } -static u32 bytes2block(struct dw_dma_chan *dwc, size_t bytes, - unsigned int width, size_t *len) -{ - struct dw_dma *dw = to_dw_dma(dwc->chan.device); - u32 block; - - /* Always in bytes for iDMA 32-bit */ - if (dw->pdata->is_idma32) - width = 0; - - if ((bytes >> width) > dwc->block_size) { - block = dwc->block_size; - *len = block << width; - } else { - block = bytes >> width; - *len = bytes; - } - - return block; -} - -static size_t block2bytes(struct dw_dma_chan *dwc, u32 block, u32 width) -{ - struct dw_dma *dw = to_dw_dma(dwc->chan.device); - - if (dw->pdata->is_idma32) - return IDMA32C_CTLH_BLOCK_TS(block); - - return DWC_CTLH_BLOCK_TS(block) << width; -} - /*----------------------------------------------------------------------*/ /* Perform single block transfer */ @@ -391,10 +295,11 @@ static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc) /* Returns how many bytes were already received from source */ static inline u32 dwc_get_sent(struct dw_dma_chan *dwc) { + struct dw_dma *dw = to_dw_dma(dwc->chan.device); u32 ctlhi = channel_readl(dwc, CTL_HI); u32 ctllo = channel_readl(dwc, CTL_LO); - return block2bytes(dwc, ctlhi, ctllo >> 4 & 7); + return dw->block2bytes(dwc, ctlhi, ctllo >> 4 & 7); } static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc) @@ -651,7 +556,7 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, unsigned int src_width; unsigned int dst_width; unsigned int data_width = dw->pdata->data_width[m_master]; - u32 ctllo; + u32 ctllo, ctlhi; u8 lms = DWC_LLP_LMS(m_master); dev_vdbg(chan2dev(chan), @@ -667,7 +572,7 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, src_width = dst_width = __ffs(data_width | src | dest | len); - ctllo = DWC_DEFAULT_CTLLO(chan) + ctllo = dw->prepare_ctllo(dwc) | DWC_CTLL_DST_WIDTH(dst_width) | DWC_CTLL_SRC_WIDTH(src_width) | DWC_CTLL_DST_INC @@ -680,10 +585,12 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, if (!desc) goto err_desc_get; + ctlhi = dw->bytes2block(dwc, len - offset, src_width, &xfer_count); + lli_write(desc, sar, src + offset); lli_write(desc, dar, dest + offset); lli_write(desc, ctllo, ctllo); - lli_write(desc, ctlhi, bytes2block(dwc, len - offset, src_width, &xfer_count)); + lli_write(desc, ctlhi, ctlhi); desc->len = xfer_count; if (!first) { @@ -721,7 +628,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, struct dma_slave_config *sconfig = &dwc->dma_sconfig; struct dw_desc *prev; struct dw_desc *first; - u32 ctllo; + u32 ctllo, ctlhi; u8 m_master = dwc->dws.m_master; u8 lms = DWC_LLP_LMS(m_master); dma_addr_t reg; @@ -745,10 +652,10 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, case DMA_MEM_TO_DEV: reg_width = __ffs(sconfig->dst_addr_width); reg = sconfig->dst_addr; - ctllo = (DWC_DEFAULT_CTLLO(chan) + ctllo = dw->prepare_ctllo(dwc) | DWC_CTLL_DST_WIDTH(reg_width) | DWC_CTLL_DST_FIX - | DWC_CTLL_SRC_INC); + | DWC_CTLL_SRC_INC; ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) : DWC_CTLL_FC(DW_DMA_FC_D_M2P); @@ -768,9 +675,11 @@ slave_sg_todev_fill_desc: if (!desc) goto err_desc_get; + ctlhi = dw->bytes2block(dwc, len, mem_width, &dlen); + lli_write(desc, sar, mem); lli_write(desc, dar, reg); - lli_write(desc, ctlhi, bytes2block(dwc, len, mem_width, &dlen)); + lli_write(desc, ctlhi, ctlhi); lli_write(desc, ctllo, ctllo | DWC_CTLL_SRC_WIDTH(mem_width)); desc->len = dlen; @@ -793,10 +702,10 @@ slave_sg_todev_fill_desc: case DMA_DEV_TO_MEM: reg_width = __ffs(sconfig->src_addr_width); reg = sconfig->src_addr; - ctllo = (DWC_DEFAULT_CTLLO(chan) + ctllo = dw->prepare_ctllo(dwc) | DWC_CTLL_SRC_WIDTH(reg_width) | DWC_CTLL_DST_INC - | DWC_CTLL_SRC_FIX); + | DWC_CTLL_SRC_FIX; ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) : DWC_CTLL_FC(DW_DMA_FC_D_P2M); @@ -814,9 +723,11 @@ slave_sg_fromdev_fill_desc: if (!desc) goto err_desc_get; + ctlhi = dw->bytes2block(dwc, len, reg_width, &dlen); + lli_write(desc, sar, reg); lli_write(desc, dar, mem); - lli_write(desc, ctlhi, bytes2block(dwc, len, reg_width, &dlen)); + lli_write(desc, ctlhi, ctlhi); mem_width = __ffs(data_width | mem | dlen); lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width)); desc->len = dlen; @@ -876,22 +787,12 @@ EXPORT_SYMBOL_GPL(dw_dma_filter); static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig) { struct dw_dma_chan *dwc = to_dw_dma_chan(chan); - struct dma_slave_config *sc = &dwc->dma_sconfig; struct dw_dma *dw = to_dw_dma(chan->device); - /* - * Fix sconfig's burst size according to dw_dmac. We need to convert - * them as: - * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3. - * - * NOTE: burst size 2 is not supported by DesignWare controller. - * iDMA 32-bit supports it. - */ - u32 s = dw->pdata->is_idma32 ? 1 : 2; memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig)); - sc->src_maxburst = sc->src_maxburst > 1 ? fls(sc->src_maxburst) - s : 0; - sc->dst_maxburst = sc->dst_maxburst > 1 ? fls(sc->dst_maxburst) - s : 0; + dw->encode_maxburst(dwc, &dwc->dma_sconfig.src_maxburst); + dw->encode_maxburst(dwc, &dwc->dma_sconfig.dst_maxburst); return 0; } @@ -900,16 +801,9 @@ static void dwc_chan_pause(struct dw_dma_chan *dwc, bool drain) { struct dw_dma *dw = to_dw_dma(dwc->chan.device); unsigned int count = 20; /* timeout iterations */ - u32 cfglo; - cfglo = channel_readl(dwc, CFG_LO); - if (dw->pdata->is_idma32) { - if (drain) - cfglo |= IDMA32C_CFGL_CH_DRAIN; - else - cfglo &= ~IDMA32C_CFGL_CH_DRAIN; - } - channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP); + dw->suspend_chan(dwc, drain); + while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY) && count--) udelay(2); @@ -928,11 +822,11 @@ static int dwc_pause(struct dma_chan *chan) return 0; } -static inline void dwc_chan_resume(struct dw_dma_chan *dwc) +static inline void dwc_chan_resume(struct dw_dma_chan *dwc, bool drain) { - u32 cfglo = channel_readl(dwc, CFG_LO); + struct dw_dma *dw = to_dw_dma(dwc->chan.device); - channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP); + dw->resume_chan(dwc, drain); clear_bit(DW_DMA_IS_PAUSED, &dwc->flags); } @@ -945,7 +839,7 @@ static int dwc_resume(struct dma_chan *chan) spin_lock_irqsave(&dwc->lock, flags); if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags)) - dwc_chan_resume(dwc); + dwc_chan_resume(dwc, false); spin_unlock_irqrestore(&dwc->lock, flags); @@ -968,7 +862,7 @@ static int dwc_terminate_all(struct dma_chan *chan) dwc_chan_disable(dw, dwc); - dwc_chan_resume(dwc); + dwc_chan_resume(dwc, true); /* active_list entries will end up before queued entries */ list_splice_init(&dwc->queue, &list); @@ -1058,33 +952,7 @@ static void dwc_issue_pending(struct dma_chan *chan) /*----------------------------------------------------------------------*/ -/* - * Program FIFO size of channels. - * - * By default full FIFO (512 bytes) is assigned to channel 0. Here we - * slice FIFO on equal parts between channels. - */ -static void idma32_fifo_partition(struct dw_dma *dw) -{ - u64 value = IDMA32C_FP_PSIZE_CH0(64) | IDMA32C_FP_PSIZE_CH1(64) | - IDMA32C_FP_UPDATE; - u64 fifo_partition = 0; - - if (!dw->pdata->is_idma32) - return; - - /* Fill FIFO_PARTITION low bits (Channels 0..1, 4..5) */ - fifo_partition |= value << 0; - - /* Fill FIFO_PARTITION high bits (Channels 2..3, 6..7) */ - fifo_partition |= value << 32; - - /* Program FIFO Partition registers - 64 bytes per channel */ - idma32_writeq(dw, FIFO_PARTITION1, fifo_partition); - idma32_writeq(dw, FIFO_PARTITION0, fifo_partition); -} - -static void dw_dma_off(struct dw_dma *dw) +void do_dw_dma_off(struct dw_dma *dw) { unsigned int i; @@ -1103,7 +971,7 @@ static void dw_dma_off(struct dw_dma *dw) clear_bit(DW_DMA_IS_INITIALIZED, &dw->chan[i].flags); } -static void dw_dma_on(struct dw_dma *dw) +void do_dw_dma_on(struct dw_dma *dw) { dma_writel(dw, CFG, DW_CFG_DMA_EN); } @@ -1139,7 +1007,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan) /* Enable controller here if needed */ if (!dw->in_use) - dw_dma_on(dw); + do_dw_dma_on(dw); dw->in_use |= dwc->mask; return 0; @@ -1150,7 +1018,6 @@ static void dwc_free_chan_resources(struct dma_chan *chan) struct dw_dma_chan *dwc = to_dw_dma_chan(chan); struct dw_dma *dw = to_dw_dma(chan->device); unsigned long flags; - LIST_HEAD(list); dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__, dwc->descs_allocated); @@ -1177,30 +1044,25 @@ static void dwc_free_chan_resources(struct dma_chan *chan) /* Disable controller in case it was a last user */ dw->in_use &= ~dwc->mask; if (!dw->in_use) - dw_dma_off(dw); + do_dw_dma_off(dw); dev_vdbg(chan2dev(chan), "%s: done\n", __func__); } -int dw_dma_probe(struct dw_dma_chip *chip) +int do_dma_probe(struct dw_dma_chip *chip) { + struct dw_dma *dw = chip->dw; struct dw_dma_platform_data *pdata; - struct dw_dma *dw; bool autocfg = false; unsigned int dw_params; unsigned int i; int err; - dw = devm_kzalloc(chip->dev, sizeof(*dw), GFP_KERNEL); - if (!dw) - return -ENOMEM; - dw->pdata = devm_kzalloc(chip->dev, sizeof(*dw->pdata), GFP_KERNEL); if (!dw->pdata) return -ENOMEM; dw->regs = chip->regs; - chip->dw = dw; pm_runtime_get_sync(chip->dev); @@ -1227,8 +1089,6 @@ int dw_dma_probe(struct dw_dma_chip *chip) pdata->block_size = dma_readl(dw, MAX_BLK_SIZE); /* Fill platform data with the default values */ - pdata->is_private = true; - pdata->is_memcpy = true; pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING; pdata->chan_priority = CHAN_PRIORITY_ASCENDING; } else if (chip->pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) { @@ -1252,15 +1112,10 @@ int dw_dma_probe(struct dw_dma_chip *chip) dw->all_chan_mask = (1 << pdata->nr_channels) - 1; /* Force dma off, just in case */ - dw_dma_off(dw); - - idma32_fifo_partition(dw); + dw->disable(dw); /* Device and instance ID for IRQ and DMA pool */ - if (pdata->is_idma32) - snprintf(dw->name, sizeof(dw->name), "idma32:dmac%d", chip->id); - else - snprintf(dw->name, sizeof(dw->name), "dw:dmac%d", chip->id); + dw->set_device_name(dw, chip->id); /* Create a pool of consistent memory blocks for hardware descriptors */ dw->desc_pool = dmam_pool_create(dw->name, chip->dev, @@ -1340,10 +1195,8 @@ int dw_dma_probe(struct dw_dma_chip *chip) /* Set capabilities */ dma_cap_set(DMA_SLAVE, dw->dma.cap_mask); - if (pdata->is_private) - dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask); - if (pdata->is_memcpy) - dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask); + dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask); + dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask); dw->dma.dev = chip->dev; dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources; @@ -1384,16 +1237,15 @@ err_pdata: pm_runtime_put_sync_suspend(chip->dev); return err; } -EXPORT_SYMBOL_GPL(dw_dma_probe); -int dw_dma_remove(struct dw_dma_chip *chip) +int do_dma_remove(struct dw_dma_chip *chip) { struct dw_dma *dw = chip->dw; struct dw_dma_chan *dwc, *_dwc; pm_runtime_get_sync(chip->dev); - dw_dma_off(dw); + do_dw_dma_off(dw); dma_async_device_unregister(&dw->dma); free_irq(chip->irq, dw); @@ -1408,27 +1260,24 @@ int dw_dma_remove(struct dw_dma_chip *chip) pm_runtime_put_sync_suspend(chip->dev); return 0; } -EXPORT_SYMBOL_GPL(dw_dma_remove); -int dw_dma_disable(struct dw_dma_chip *chip) +int do_dw_dma_disable(struct dw_dma_chip *chip) { struct dw_dma *dw = chip->dw; - dw_dma_off(dw); + dw->disable(dw); return 0; } -EXPORT_SYMBOL_GPL(dw_dma_disable); +EXPORT_SYMBOL_GPL(do_dw_dma_disable); -int dw_dma_enable(struct dw_dma_chip *chip) +int do_dw_dma_enable(struct dw_dma_chip *chip) { struct dw_dma *dw = chip->dw; - idma32_fifo_partition(dw); - - dw_dma_on(dw); + dw->enable(dw); return 0; } -EXPORT_SYMBOL_GPL(dw_dma_enable); +EXPORT_SYMBOL_GPL(do_dw_dma_enable); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller core driver"); diff --git a/drivers/dma/dw/dw.c b/drivers/dma/dw/dw.c new file mode 100644 index 0000000000000..7a085b3c1854c --- /dev/null +++ b/drivers/dma/dw/dw.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2007-2008 Atmel Corporation +// Copyright (C) 2010-2011 ST Microelectronics +// Copyright (C) 2013,2018 Intel Corporation + +#include <linux/bitops.h> +#include <linux/dmaengine.h> +#include <linux/errno.h> +#include <linux/slab.h> +#include <linux/types.h> + +#include "internal.h" + +static void dw_dma_initialize_chan(struct dw_dma_chan *dwc) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + u32 cfghi = DWC_CFGH_FIFO_MODE; + u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority); + bool hs_polarity = dwc->dws.hs_polarity; + + cfghi |= DWC_CFGH_DST_PER(dwc->dws.dst_id); + cfghi |= DWC_CFGH_SRC_PER(dwc->dws.src_id); + cfghi |= DWC_CFGH_PROTCTL(dw->pdata->protctl); + + /* Set polarity of handshake interface */ + cfglo |= hs_polarity ? DWC_CFGL_HS_DST_POL | DWC_CFGL_HS_SRC_POL : 0; + + channel_writel(dwc, CFG_LO, cfglo); + channel_writel(dwc, CFG_HI, cfghi); +} + +static void dw_dma_suspend_chan(struct dw_dma_chan *dwc, bool drain) +{ + u32 cfglo = channel_readl(dwc, CFG_LO); + + channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP); +} + +static void dw_dma_resume_chan(struct dw_dma_chan *dwc, bool drain) +{ + u32 cfglo = channel_readl(dwc, CFG_LO); + + channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP); +} + +static u32 dw_dma_bytes2block(struct dw_dma_chan *dwc, + size_t bytes, unsigned int width, size_t *len) +{ + u32 block; + + if ((bytes >> width) > dwc->block_size) { + block = dwc->block_size; + *len = dwc->block_size << width; + } else { + block = bytes >> width; + *len = bytes; + } + + return block; +} + +static size_t dw_dma_block2bytes(struct dw_dma_chan *dwc, u32 block, u32 width) +{ + return DWC_CTLH_BLOCK_TS(block) << width; +} + +static u32 dw_dma_prepare_ctllo(struct dw_dma_chan *dwc) +{ + struct dma_slave_config *sconfig = &dwc->dma_sconfig; + bool is_slave = is_slave_direction(dwc->direction); + u8 smsize = is_slave ? sconfig->src_maxburst : DW_DMA_MSIZE_16; + u8 dmsize = is_slave ? sconfig->dst_maxburst : DW_DMA_MSIZE_16; + u8 p_master = dwc->dws.p_master; + u8 m_master = dwc->dws.m_master; + u8 dms = (dwc->direction == DMA_MEM_TO_DEV) ? p_master : m_master; + u8 sms = (dwc->direction == DMA_DEV_TO_MEM) ? p_master : m_master; + + return DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN | + DWC_CTLL_DST_MSIZE(dmsize) | DWC_CTLL_SRC_MSIZE(smsize) | + DWC_CTLL_DMS(dms) | DWC_CTLL_SMS(sms); +} + +static void dw_dma_encode_maxburst(struct dw_dma_chan *dwc, u32 *maxburst) +{ + /* + * Fix burst size according to dw_dmac. We need to convert them as: + * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3. + */ + *maxburst = *maxburst > 1 ? fls(*maxburst) - 2 : 0; +} + +static void dw_dma_set_device_name(struct dw_dma *dw, int id) +{ + snprintf(dw->name, sizeof(dw->name), "dw:dmac%d", id); +} + +static void dw_dma_disable(struct dw_dma *dw) +{ + do_dw_dma_off(dw); +} + +static void dw_dma_enable(struct dw_dma *dw) +{ + do_dw_dma_on(dw); +} + +int dw_dma_probe(struct dw_dma_chip *chip) +{ + struct dw_dma *dw; + + dw = devm_kzalloc(chip->dev, sizeof(*dw), GFP_KERNEL); + if (!dw) + return -ENOMEM; + + /* Channel operations */ + dw->initialize_chan = dw_dma_initialize_chan; + dw->suspend_chan = dw_dma_suspend_chan; + dw->resume_chan = dw_dma_resume_chan; + dw->prepare_ctllo = dw_dma_prepare_ctllo; + dw->encode_maxburst = dw_dma_encode_maxburst; + dw->bytes2block = dw_dma_bytes2block; + dw->block2bytes = dw_dma_block2bytes; + + /* Device operations */ + dw->set_device_name = dw_dma_set_device_name; + dw->disable = dw_dma_disable; + dw->enable = dw_dma_enable; + + chip->dw = dw; + return do_dma_probe(chip); +} +EXPORT_SYMBOL_GPL(dw_dma_probe); + +int dw_dma_remove(struct dw_dma_chip *chip) +{ + return do_dma_remove(chip); +} +EXPORT_SYMBOL_GPL(dw_dma_remove); diff --git a/drivers/dma/dw/idma32.c b/drivers/dma/dw/idma32.c new file mode 100644 index 0000000000000..f006573088110 --- /dev/null +++ b/drivers/dma/dw/idma32.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2013,2018 Intel Corporation + +#include <linux/bitops.h> +#include <linux/dmaengine.h> +#include <linux/errno.h> +#include <linux/slab.h> +#include <linux/types.h> + +#include "internal.h" + +static void idma32_initialize_chan(struct dw_dma_chan *dwc) +{ + u32 cfghi = 0; + u32 cfglo = 0; + + /* Set default burst alignment */ + cfglo |= IDMA32C_CFGL_DST_BURST_ALIGN | IDMA32C_CFGL_SRC_BURST_ALIGN; + + /* Low 4 bits of the request lines */ + cfghi |= IDMA32C_CFGH_DST_PER(dwc->dws.dst_id & 0xf); + cfghi |= IDMA32C_CFGH_SRC_PER(dwc->dws.src_id & 0xf); + + /* Request line extension (2 bits) */ + cfghi |= IDMA32C_CFGH_DST_PER_EXT(dwc->dws.dst_id >> 4 & 0x3); + cfghi |= IDMA32C_CFGH_SRC_PER_EXT(dwc->dws.src_id >> 4 & 0x3); + + channel_writel(dwc, CFG_LO, cfglo); + channel_writel(dwc, CFG_HI, cfghi); +} + +static void idma32_suspend_chan(struct dw_dma_chan *dwc, bool drain) +{ + u32 cfglo = channel_readl(dwc, CFG_LO); + + if (drain) + cfglo |= IDMA32C_CFGL_CH_DRAIN; + + channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP); +} + +static void idma32_resume_chan(struct dw_dma_chan *dwc, bool drain) +{ + u32 cfglo = channel_readl(dwc, CFG_LO); + + if (drain) + cfglo &= ~IDMA32C_CFGL_CH_DRAIN; + + channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP); +} + +static u32 idma32_bytes2block(struct dw_dma_chan *dwc, + size_t bytes, unsigned int width, size_t *len) +{ + u32 block; + + if (bytes > dwc->block_size) { + block = dwc->block_size; + *len = dwc->block_size; + } else { + block = bytes; + *len = bytes; + } + + return block; +} + +static size_t idma32_block2bytes(struct dw_dma_chan *dwc, u32 block, u32 width) +{ + return IDMA32C_CTLH_BLOCK_TS(block); +} + +static u32 idma32_prepare_ctllo(struct dw_dma_chan *dwc) +{ + struct dma_slave_config *sconfig = &dwc->dma_sconfig; + bool is_slave = is_slave_direction(dwc->direction); + u8 smsize = is_slave ? sconfig->src_maxburst : IDMA32_MSIZE_8; + u8 dmsize = is_slave ? sconfig->dst_maxburst : IDMA32_MSIZE_8; + + return DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN | + DWC_CTLL_DST_MSIZE(dmsize) | DWC_CTLL_SRC_MSIZE(smsize); +} + +static void idma32_encode_maxburst(struct dw_dma_chan *dwc, u32 *maxburst) +{ + *maxburst = *maxburst > 1 ? fls(*maxburst) - 1 : 0; +} + +static void idma32_set_device_name(struct dw_dma *dw, int id) +{ + snprintf(dw->name, sizeof(dw->name), "idma32:dmac%d", id); +} + +/* + * Program FIFO size of channels. + * + * By default full FIFO (512 bytes) is assigned to channel 0. Here we + * slice FIFO on equal parts between channels. + */ +static void idma32_fifo_partition(struct dw_dma *dw) +{ + u64 value = IDMA32C_FP_PSIZE_CH0(64) | IDMA32C_FP_PSIZE_CH1(64) | + IDMA32C_FP_UPDATE; + u64 fifo_partition = 0; + + /* Fill FIFO_PARTITION low bits (Channels 0..1, 4..5) */ + fifo_partition |= value << 0; + + /* Fill FIFO_PARTITION high bits (Channels 2..3, 6..7) */ + fifo_partition |= value << 32; + + /* Program FIFO Partition registers - 64 bytes per channel */ + idma32_writeq(dw, FIFO_PARTITION1, fifo_partition); + idma32_writeq(dw, FIFO_PARTITION0, fifo_partition); +} + +static void idma32_disable(struct dw_dma *dw) +{ + do_dw_dma_off(dw); + idma32_fifo_partition(dw); +} + +static void idma32_enable(struct dw_dma *dw) +{ + idma32_fifo_partition(dw); + do_dw_dma_on(dw); +} + +int idma32_dma_probe(struct dw_dma_chip *chip) +{ + struct dw_dma *dw; + + dw = devm_kzalloc(chip->dev, sizeof(*dw), GFP_KERNEL); + if (!dw) + return -ENOMEM; + + /* Channel operations */ + dw->initialize_chan = idma32_initialize_chan; + dw->suspend_chan = idma32_suspend_chan; + dw->resume_chan = idma32_resume_chan; + dw->prepare_ctllo = idma32_prepare_ctllo; + dw->encode_maxburst = idma32_encode_maxburst; + dw->bytes2block = idma32_bytes2block; + dw->block2bytes = idma32_block2bytes; + + /* Device operations */ + dw->set_device_name = idma32_set_device_name; + dw->disable = idma32_disable; + dw->enable = idma32_enable; + + chip->dw = dw; + return do_dma_probe(chip); +} +EXPORT_SYMBOL_GPL(idma32_dma_probe); + +int idma32_dma_remove(struct dw_dma_chip *chip) +{ + return do_dma_remove(chip); +} +EXPORT_SYMBOL_GPL(idma32_dma_remove); diff --git a/drivers/dma/dw/internal.h b/drivers/dma/dw/internal.h index 41439732ff6b8..1dd7a4e6dd235 100644 --- a/drivers/dma/dw/internal.h +++ b/drivers/dma/dw/internal.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Driver for the Synopsys DesignWare DMA Controller * * Copyright (C) 2013 Intel Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef _DMA_DW_INTERNAL_H @@ -15,8 +12,14 @@ #include "regs.h" -int dw_dma_disable(struct dw_dma_chip *chip); -int dw_dma_enable(struct dw_dma_chip *chip); +int do_dma_probe(struct dw_dma_chip *chip); +int do_dma_remove(struct dw_dma_chip *chip); + +void do_dw_dma_on(struct dw_dma *dw); +void do_dw_dma_off(struct dw_dma *dw); + +int do_dw_dma_disable(struct dw_dma_chip *chip); +int do_dw_dma_enable(struct dw_dma_chip *chip); extern bool dw_dma_filter(struct dma_chan *chan, void *param); diff --git a/drivers/dma/dw/pci.c b/drivers/dma/dw/pci.c index 7778ed705a1ad..e79a75db08526 100644 --- a/drivers/dma/dw/pci.c +++ b/drivers/dma/dw/pci.c @@ -1,12 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * PCI driver for the Synopsys DesignWare DMA Controller * * Copyright (C) 2013 Intel Corporation * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include <linux/module.h> @@ -15,21 +12,33 @@ #include "internal.h" -static struct dw_dma_platform_data mrfld_pdata = { +struct dw_dma_pci_data { + const struct dw_dma_platform_data *pdata; + int (*probe)(struct dw_dma_chip *chip); +}; + +static const struct dw_dma_pci_data dw_pci_data = { + .probe = dw_dma_probe, +}; + +static const struct dw_dma_platform_data idma32_pdata = { .nr_channels = 8, - .is_private = true, - .is_memcpy = true, - .is_idma32 = true, .chan_allocation_order = CHAN_ALLOCATION_ASCENDING, .chan_priority = CHAN_PRIORITY_ASCENDING, .block_size = 131071, .nr_masters = 1, .data_width = {4}, + .multi_block = {1, 1, 1, 1, 1, 1, 1, 1}, +}; + +static const struct dw_dma_pci_data idma32_pci_data = { + .pdata = &idma32_pdata, + .probe = idma32_dma_probe, }; static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) { - const struct dw_dma_platform_data *pdata = (void *)pid->driver_data; + const struct dw_dma_pci_data *data = (void *)pid->driver_data; struct dw_dma_chip *chip; int ret; @@ -62,9 +71,9 @@ static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) chip->id = pdev->devfn; chip->regs = pcim_iomap_table(pdev)[0]; chip->irq = pdev->irq; - chip->pdata = pdata; + chip->pdata = data->pdata; - ret = dw_dma_probe(chip); + ret = data->probe(chip); if (ret) return ret; @@ -90,7 +99,7 @@ static int dw_pci_suspend_late(struct device *dev) struct pci_dev *pci = to_pci_dev(dev); struct dw_dma_chip *chip = pci_get_drvdata(pci); - return dw_dma_disable(chip); + return do_dw_dma_disable(chip); }; static int dw_pci_resume_early(struct device *dev) @@ -98,7 +107,7 @@ static int dw_pci_resume_early(struct device *dev) struct pci_dev *pci = to_pci_dev(dev); struct dw_dma_chip *chip = pci_get_drvdata(pci); - return dw_dma_enable(chip); + return do_dw_dma_enable(chip); }; #endif /* CONFIG_PM_SLEEP */ @@ -109,24 +118,24 @@ static const struct dev_pm_ops dw_pci_dev_pm_ops = { static const struct pci_device_id dw_pci_id_table[] = { /* Medfield (GPDMA) */ - { PCI_VDEVICE(INTEL, 0x0827) }, + { PCI_VDEVICE(INTEL, 0x0827), (kernel_ulong_t)&dw_pci_data }, /* BayTrail */ - { PCI_VDEVICE(INTEL, 0x0f06) }, - { PCI_VDEVICE(INTEL, 0x0f40) }, + { PCI_VDEVICE(INTEL, 0x0f06), (kernel_ulong_t)&dw_pci_data }, + { PCI_VDEVICE(INTEL, 0x0f40), (kernel_ulong_t)&dw_pci_data }, - /* Merrifield iDMA 32-bit (GPDMA) */ - { PCI_VDEVICE(INTEL, 0x11a2), (kernel_ulong_t)&mrfld_pdata }, + /* Merrifield */ + { PCI_VDEVICE(INTEL, 0x11a2), (kernel_ulong_t)&idma32_pci_data }, /* Braswell */ - { PCI_VDEVICE(INTEL, 0x2286) }, - { PCI_VDEVICE(INTEL, 0x22c0) }, + { PCI_VDEVICE(INTEL, 0x2286), (kernel_ulong_t)&dw_pci_data }, + { PCI_VDEVICE(INTEL, 0x22c0), (kernel_ulong_t)&dw_pci_data }, /* Haswell */ - { PCI_VDEVICE(INTEL, 0x9c60) }, + { PCI_VDEVICE(INTEL, 0x9c60), (kernel_ulong_t)&dw_pci_data }, /* Broadwell */ - { PCI_VDEVICE(INTEL, 0x9ce0) }, + { PCI_VDEVICE(INTEL, 0x9ce0), (kernel_ulong_t)&dw_pci_data }, { } }; diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c index 31ff8113c3def..382dfd9e9600e 100644 --- a/drivers/dma/dw/platform.c +++ b/drivers/dma/dw/platform.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Platform driver for the Synopsys DesignWare DMA Controller * @@ -6,10 +7,6 @@ * Copyright (C) 2013 Intel Corporation * * Some parts of this driver are derived from the original dw_dmac. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include <linux/module.h> @@ -128,15 +125,6 @@ dw_dma_parse_dt(struct platform_device *pdev) pdata->nr_masters = nr_masters; pdata->nr_channels = nr_channels; - if (of_property_read_bool(np, "is_private")) - pdata->is_private = true; - - /* - * All known devices, which use DT for configuration, support - * memory-to-memory transfers. So enable it by default. - */ - pdata->is_memcpy = true; - if (!of_property_read_u32(np, "chan_allocation_order", &tmp)) pdata->chan_allocation_order = (unsigned char)tmp; @@ -264,7 +252,7 @@ static void dw_shutdown(struct platform_device *pdev) struct dw_dma_chip *chip = platform_get_drvdata(pdev); /* - * We have to call dw_dma_disable() to stop any ongoing transfer. On + * We have to call do_dw_dma_disable() to stop any ongoing transfer. On * some platforms we can't do that since DMA device is powered off. * Moreover we have no possibility to check if the platform is affected * or not. That's why we call pm_runtime_get_sync() / pm_runtime_put() @@ -273,7 +261,7 @@ static void dw_shutdown(struct platform_device *pdev) * used by the driver. */ pm_runtime_get_sync(chip->dev); - dw_dma_disable(chip); + do_dw_dma_disable(chip); pm_runtime_put_sync_suspend(chip->dev); clk_disable_unprepare(chip->clk); @@ -303,7 +291,7 @@ static int dw_suspend_late(struct device *dev) { struct dw_dma_chip *chip = dev_get_drvdata(dev); - dw_dma_disable(chip); + do_dw_dma_disable(chip); clk_disable_unprepare(chip->clk); return 0; @@ -318,7 +306,7 @@ static int dw_resume_early(struct device *dev) if (ret) return ret; - return dw_dma_enable(chip); + return do_dw_dma_enable(chip); } #endif /* CONFIG_PM_SLEEP */ diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h index 646c9c960c071..3fce66ecee7a5 100644 --- a/drivers/dma/dw/regs.h +++ b/drivers/dma/dw/regs.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Driver for the Synopsys DesignWare AHB DMA Controller * * Copyright (C) 2005-2007 Atmel Corporation * Copyright (C) 2010-2011 ST Microelectronics * Copyright (C) 2016 Intel Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include <linux/bitops.h> @@ -222,6 +219,16 @@ enum dw_dma_msize { /* iDMA 32-bit support */ +/* bursts size */ +enum idma32_msize { + IDMA32_MSIZE_1, + IDMA32_MSIZE_2, + IDMA32_MSIZE_4, + IDMA32_MSIZE_8, + IDMA32_MSIZE_16, + IDMA32_MSIZE_32, +}; + /* Bitfields in CTL_HI */ #define IDMA32C_CTLH_BLOCK_TS_MASK GENMASK(16, 0) #define IDMA32C_CTLH_BLOCK_TS(x) ((x) & IDMA32C_CTLH_BLOCK_TS_MASK) @@ -312,6 +319,21 @@ struct dw_dma { u8 all_chan_mask; u8 in_use; + /* Channel operations */ + void (*initialize_chan)(struct dw_dma_chan *dwc); + void (*suspend_chan)(struct dw_dma_chan *dwc, bool drain); + void (*resume_chan)(struct dw_dma_chan *dwc, bool drain); + u32 (*prepare_ctllo)(struct dw_dma_chan *dwc); + void (*encode_maxburst)(struct dw_dma_chan *dwc, u32 *maxburst); + u32 (*bytes2block)(struct dw_dma_chan *dwc, size_t bytes, + unsigned int width, size_t *len); + size_t (*block2bytes)(struct dw_dma_chan *dwc, u32 block, u32 width); + + /* Device operations */ + void (*set_device_name)(struct dw_dma *dw, int id); + void (*disable)(struct dw_dma *dw); + void (*enable)(struct dw_dma *dw); + /* platform data */ struct dw_dma_platform_data *pdata; }; diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c index 8876c4c1bb2c0..680b2a00a9532 100644 --- a/drivers/dma/fsl-edma-common.c +++ b/drivers/dma/fsl-edma-common.c @@ -6,6 +6,7 @@ #include <linux/dmapool.h> #include <linux/module.h> #include <linux/slab.h> +#include <linux/dma-mapping.h> #include "fsl-edma-common.h" @@ -173,12 +174,62 @@ int fsl_edma_resume(struct dma_chan *chan) } EXPORT_SYMBOL_GPL(fsl_edma_resume); +static void fsl_edma_unprep_slave_dma(struct fsl_edma_chan *fsl_chan) +{ + if (fsl_chan->dma_dir != DMA_NONE) + dma_unmap_resource(fsl_chan->vchan.chan.device->dev, + fsl_chan->dma_dev_addr, + fsl_chan->dma_dev_size, + fsl_chan->dma_dir, 0); + fsl_chan->dma_dir = DMA_NONE; +} + +static bool fsl_edma_prep_slave_dma(struct fsl_edma_chan *fsl_chan, + enum dma_transfer_direction dir) +{ + struct device *dev = fsl_chan->vchan.chan.device->dev; + enum dma_data_direction dma_dir; + phys_addr_t addr = 0; + u32 size = 0; + + switch (dir) { + case DMA_MEM_TO_DEV: + dma_dir = DMA_FROM_DEVICE; + addr = fsl_chan->cfg.dst_addr; + size = fsl_chan->cfg.dst_maxburst; + break; + case DMA_DEV_TO_MEM: + dma_dir = DMA_TO_DEVICE; + addr = fsl_chan->cfg.src_addr; + size = fsl_chan->cfg.src_maxburst; + break; + default: + dma_dir = DMA_NONE; + break; + } + + /* Already mapped for this config? */ + if (fsl_chan->dma_dir == dma_dir) + return true; + + fsl_edma_unprep_slave_dma(fsl_chan); + + fsl_chan->dma_dev_addr = dma_map_resource(dev, addr, size, dma_dir, 0); + if (dma_mapping_error(dev, fsl_chan->dma_dev_addr)) + return false; + fsl_chan->dma_dev_size = size; + fsl_chan->dma_dir = dma_dir; + + return true; +} + int fsl_edma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) { struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); memcpy(&fsl_chan->cfg, cfg, sizeof(*cfg)); + fsl_edma_unprep_slave_dma(fsl_chan); return 0; } @@ -339,9 +390,7 @@ static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan, struct fsl_edma_desc *fsl_desc; int i; - fsl_desc = kzalloc(sizeof(*fsl_desc) + - sizeof(struct fsl_edma_sw_tcd) * - sg_len, GFP_NOWAIT); + fsl_desc = kzalloc(struct_size(fsl_desc, tcd, sg_len), GFP_NOWAIT); if (!fsl_desc) return NULL; @@ -378,6 +427,9 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( if (!is_slave_direction(direction)) return NULL; + if (!fsl_edma_prep_slave_dma(fsl_chan, direction)) + return NULL; + sg_len = buf_len / period_len; fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); if (!fsl_desc) @@ -409,11 +461,11 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( if (direction == DMA_MEM_TO_DEV) { src_addr = dma_buf_next; - dst_addr = fsl_chan->cfg.dst_addr; + dst_addr = fsl_chan->dma_dev_addr; soff = fsl_chan->cfg.dst_addr_width; doff = 0; } else { - src_addr = fsl_chan->cfg.src_addr; + src_addr = fsl_chan->dma_dev_addr; dst_addr = dma_buf_next; soff = 0; doff = fsl_chan->cfg.src_addr_width; @@ -444,6 +496,9 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( if (!is_slave_direction(direction)) return NULL; + if (!fsl_edma_prep_slave_dma(fsl_chan, direction)) + return NULL; + fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); if (!fsl_desc) return NULL; @@ -468,11 +523,11 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( if (direction == DMA_MEM_TO_DEV) { src_addr = sg_dma_address(sg); - dst_addr = fsl_chan->cfg.dst_addr; + dst_addr = fsl_chan->dma_dev_addr; soff = fsl_chan->cfg.dst_addr_width; doff = 0; } else { - src_addr = fsl_chan->cfg.src_addr; + src_addr = fsl_chan->dma_dev_addr; dst_addr = sg_dma_address(sg); soff = 0; doff = fsl_chan->cfg.src_addr_width; @@ -555,6 +610,7 @@ void fsl_edma_free_chan_resources(struct dma_chan *chan) fsl_edma_chan_mux(fsl_chan, 0, false); fsl_chan->edesc = NULL; vchan_get_all_descriptors(&fsl_chan->vchan, &head); + fsl_edma_unprep_slave_dma(fsl_chan); spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); vchan_dma_desc_free_list(&fsl_chan->vchan, &head); diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h index 8917e8865959a..b435d8e1e3a19 100644 --- a/drivers/dma/fsl-edma-common.h +++ b/drivers/dma/fsl-edma-common.h @@ -6,6 +6,7 @@ #ifndef _FSL_EDMA_COMMON_H_ #define _FSL_EDMA_COMMON_H_ +#include <linux/dma-direction.h> #include "virt-dma.h" #define EDMA_CR_EDBG BIT(1) @@ -120,6 +121,9 @@ struct fsl_edma_chan { struct dma_slave_config cfg; u32 attr; struct dma_pool *tcd_pool; + dma_addr_t dma_dev_addr; + u32 dma_dev_size; + enum dma_data_direction dma_dir; }; struct fsl_edma_desc { diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c index 34d70112fcc9d..75e8a7ba3a225 100644 --- a/drivers/dma/fsl-edma.c +++ b/drivers/dma/fsl-edma.c @@ -254,6 +254,7 @@ static int fsl_edma_probe(struct platform_device *pdev) fsl_chan->pm_state = RUNNING; fsl_chan->slave_id = 0; fsl_chan->idle = true; + fsl_chan->dma_dir = DMA_NONE; fsl_chan->vchan.desc_free = fsl_edma_free_desc; vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev); diff --git a/drivers/dma/fsl-qdma.c b/drivers/dma/fsl-qdma.c new file mode 100644 index 0000000000000..aa1d0ae3d207e --- /dev/null +++ b/drivers/dma/fsl-qdma.c @@ -0,0 +1,1259 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright 2014-2015 Freescale +// Copyright 2018 NXP + +/* + * Driver for NXP Layerscape Queue Direct Memory Access Controller + * + * Author: + * Wen He <wen.he_1@nxp.com> + * Jiaheng Fan <jiaheng.fan@nxp.com> + * + */ + +#include <linux/module.h> +#include <linux/delay.h> +#include <linux/of_irq.h> +#include <linux/of_platform.h> +#include <linux/of_dma.h> +#include <linux/dma-mapping.h> + +#include "virt-dma.h" +#include "fsldma.h" + +/* Register related definition */ +#define FSL_QDMA_DMR 0x0 +#define FSL_QDMA_DSR 0x4 +#define FSL_QDMA_DEIER 0xe00 +#define FSL_QDMA_DEDR 0xe04 +#define FSL_QDMA_DECFDW0R 0xe10 +#define FSL_QDMA_DECFDW1R 0xe14 +#define FSL_QDMA_DECFDW2R 0xe18 +#define FSL_QDMA_DECFDW3R 0xe1c +#define FSL_QDMA_DECFQIDR 0xe30 +#define FSL_QDMA_DECBR 0xe34 + +#define FSL_QDMA_BCQMR(x) (0xc0 + 0x100 * (x)) +#define FSL_QDMA_BCQSR(x) (0xc4 + 0x100 * (x)) +#define FSL_QDMA_BCQEDPA_SADDR(x) (0xc8 + 0x100 * (x)) +#define FSL_QDMA_BCQDPA_SADDR(x) (0xcc + 0x100 * (x)) +#define FSL_QDMA_BCQEEPA_SADDR(x) (0xd0 + 0x100 * (x)) +#define FSL_QDMA_BCQEPA_SADDR(x) (0xd4 + 0x100 * (x)) +#define FSL_QDMA_BCQIER(x) (0xe0 + 0x100 * (x)) +#define FSL_QDMA_BCQIDR(x) (0xe4 + 0x100 * (x)) + +#define FSL_QDMA_SQDPAR 0x80c +#define FSL_QDMA_SQEPAR 0x814 +#define FSL_QDMA_BSQMR 0x800 +#define FSL_QDMA_BSQSR 0x804 +#define FSL_QDMA_BSQICR 0x828 +#define FSL_QDMA_CQMR 0xa00 +#define FSL_QDMA_CQDSCR1 0xa08 +#define FSL_QDMA_CQDSCR2 0xa0c +#define FSL_QDMA_CQIER 0xa10 +#define FSL_QDMA_CQEDR 0xa14 +#define FSL_QDMA_SQCCMR 0xa20 + +/* Registers for bit and genmask */ +#define FSL_QDMA_CQIDR_SQT BIT(15) +#define QDMA_CCDF_FOTMAT BIT(29) +#define QDMA_CCDF_SER BIT(30) +#define QDMA_SG_FIN BIT(30) +#define QDMA_SG_LEN_MASK GENMASK(29, 0) +#define QDMA_CCDF_MASK GENMASK(28, 20) + +#define FSL_QDMA_DEDR_CLEAR GENMASK(31, 0) +#define FSL_QDMA_BCQIDR_CLEAR GENMASK(31, 0) +#define FSL_QDMA_DEIER_CLEAR GENMASK(31, 0) + +#define FSL_QDMA_BCQIER_CQTIE BIT(15) +#define FSL_QDMA_BCQIER_CQPEIE BIT(23) +#define FSL_QDMA_BSQICR_ICEN BIT(31) + +#define FSL_QDMA_BSQICR_ICST(x) ((x) << 16) +#define FSL_QDMA_CQIER_MEIE BIT(31) +#define FSL_QDMA_CQIER_TEIE BIT(0) +#define FSL_QDMA_SQCCMR_ENTER_WM BIT(21) + +#define FSL_QDMA_BCQMR_EN BIT(31) +#define FSL_QDMA_BCQMR_EI BIT(30) +#define FSL_QDMA_BCQMR_CD_THLD(x) ((x) << 20) +#define FSL_QDMA_BCQMR_CQ_SIZE(x) ((x) << 16) + +#define FSL_QDMA_BCQSR_QF BIT(16) +#define FSL_QDMA_BCQSR_XOFF BIT(0) + +#define FSL_QDMA_BSQMR_EN BIT(31) +#define FSL_QDMA_BSQMR_DI BIT(30) +#define FSL_QDMA_BSQMR_CQ_SIZE(x) ((x) << 16) + +#define FSL_QDMA_BSQSR_QE BIT(17) + +#define FSL_QDMA_DMR_DQD BIT(30) +#define FSL_QDMA_DSR_DB BIT(31) + +/* Size related definition */ +#define FSL_QDMA_QUEUE_MAX 8 +#define FSL_QDMA_COMMAND_BUFFER_SIZE 64 +#define FSL_QDMA_DESCRIPTOR_BUFFER_SIZE 32 +#define FSL_QDMA_CIRCULAR_DESC_SIZE_MIN 64 +#define FSL_QDMA_CIRCULAR_DESC_SIZE_MAX 16384 +#define FSL_QDMA_QUEUE_NUM_MAX 8 + +/* Field definition for CMD */ +#define FSL_QDMA_CMD_RWTTYPE 0x4 +#define FSL_QDMA_CMD_LWC 0x2 +#define FSL_QDMA_CMD_RWTTYPE_OFFSET 28 +#define FSL_QDMA_CMD_NS_OFFSET 27 +#define FSL_QDMA_CMD_DQOS_OFFSET 24 +#define FSL_QDMA_CMD_WTHROTL_OFFSET 20 +#define FSL_QDMA_CMD_DSEN_OFFSET 19 +#define FSL_QDMA_CMD_LWC_OFFSET 16 + +/* Field definition for Descriptor offset */ +#define QDMA_CCDF_STATUS 20 +#define QDMA_CCDF_OFFSET 20 + +/* Field definition for safe loop count*/ +#define FSL_QDMA_HALT_COUNT 1500 +#define FSL_QDMA_MAX_SIZE 16385 +#define FSL_QDMA_COMP_TIMEOUT 1000 +#define FSL_COMMAND_QUEUE_OVERFLLOW 10 + +#define FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma_engine, x) \ + (((fsl_qdma_engine)->block_offset) * (x)) + +/** + * struct fsl_qdma_format - This is the struct holding describing compound + * descriptor format with qDMA. + * @status: Command status and enqueue status notification. + * @cfg: Frame offset and frame format. + * @addr_lo: Holding the compound descriptor of the lower + * 32-bits address in memory 40-bit address. + * @addr_hi: Same as above member, but point high 8-bits in + * memory 40-bit address. + * @__reserved1: Reserved field. + * @cfg8b_w1: Compound descriptor command queue origin produced + * by qDMA and dynamic debug field. + * @data Pointer to the memory 40-bit address, describes DMA + * source information and DMA destination information. + */ +struct fsl_qdma_format { + __le32 status; + __le32 cfg; + union { + struct { + __le32 addr_lo; + u8 addr_hi; + u8 __reserved1[2]; + u8 cfg8b_w1; + } __packed; + __le64 data; + }; +} __packed; + +/* qDMA status notification pre information */ +struct fsl_pre_status { + u64 addr; + u8 queue; +}; + +static DEFINE_PER_CPU(struct fsl_pre_status, pre); + +struct fsl_qdma_chan { + struct virt_dma_chan vchan; + struct virt_dma_desc vdesc; + enum dma_status status; + struct fsl_qdma_engine *qdma; + struct fsl_qdma_queue *queue; +}; + +struct fsl_qdma_queue { + struct fsl_qdma_format *virt_head; + struct fsl_qdma_format *virt_tail; + struct list_head comp_used; + struct list_head comp_free; + struct dma_pool *comp_pool; + struct dma_pool *desc_pool; + spinlock_t queue_lock; + dma_addr_t bus_addr; + u32 n_cq; + u32 id; + struct fsl_qdma_format *cq; + void __iomem *block_base; +}; + +struct fsl_qdma_comp { + dma_addr_t bus_addr; + dma_addr_t desc_bus_addr; + struct fsl_qdma_format *virt_addr; + struct fsl_qdma_format *desc_virt_addr; + struct fsl_qdma_chan *qchan; + struct virt_dma_desc vdesc; + struct list_head list; +}; + +struct fsl_qdma_engine { + struct dma_device dma_dev; + void __iomem *ctrl_base; + void __iomem *status_base; + void __iomem *block_base; + u32 n_chans; + u32 n_queues; + struct mutex fsl_qdma_mutex; + int error_irq; + int *queue_irq; + u32 feature; + struct fsl_qdma_queue *queue; + struct fsl_qdma_queue **status; + struct fsl_qdma_chan *chans; + int block_number; + int block_offset; + int irq_base; + int desc_allocated; + +}; + +static inline u64 +qdma_ccdf_addr_get64(const struct fsl_qdma_format *ccdf) +{ + return le64_to_cpu(ccdf->data) & (U64_MAX >> 24); +} + +static inline void +qdma_desc_addr_set64(struct fsl_qdma_format *ccdf, u64 addr) +{ + ccdf->addr_hi = upper_32_bits(addr); + ccdf->addr_lo = cpu_to_le32(lower_32_bits(addr)); +} + +static inline u8 +qdma_ccdf_get_queue(const struct fsl_qdma_format *ccdf) +{ + return ccdf->cfg8b_w1 & U8_MAX; +} + +static inline int +qdma_ccdf_get_offset(const struct fsl_qdma_format *ccdf) +{ + return (le32_to_cpu(ccdf->cfg) & QDMA_CCDF_MASK) >> QDMA_CCDF_OFFSET; +} + +static inline void +qdma_ccdf_set_format(struct fsl_qdma_format *ccdf, int offset) +{ + ccdf->cfg = cpu_to_le32(QDMA_CCDF_FOTMAT | offset); +} + +static inline int +qdma_ccdf_get_status(const struct fsl_qdma_format *ccdf) +{ + return (le32_to_cpu(ccdf->status) & QDMA_CCDF_MASK) >> QDMA_CCDF_STATUS; +} + +static inline void +qdma_ccdf_set_ser(struct fsl_qdma_format *ccdf, int status) +{ + ccdf->status = cpu_to_le32(QDMA_CCDF_SER | status); +} + +static inline void qdma_csgf_set_len(struct fsl_qdma_format *csgf, int len) +{ + csgf->cfg = cpu_to_le32(len & QDMA_SG_LEN_MASK); +} + +static inline void qdma_csgf_set_f(struct fsl_qdma_format *csgf, int len) +{ + csgf->cfg = cpu_to_le32(QDMA_SG_FIN | (len & QDMA_SG_LEN_MASK)); +} + +static u32 qdma_readl(struct fsl_qdma_engine *qdma, void __iomem *addr) +{ + return FSL_DMA_IN(qdma, addr, 32); +} + +static void qdma_writel(struct fsl_qdma_engine *qdma, u32 val, + void __iomem *addr) +{ + FSL_DMA_OUT(qdma, addr, val, 32); +} + +static struct fsl_qdma_chan *to_fsl_qdma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct fsl_qdma_chan, vchan.chan); +} + +static struct fsl_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd) +{ + return container_of(vd, struct fsl_qdma_comp, vdesc); +} + +static void fsl_qdma_free_chan_resources(struct dma_chan *chan) +{ + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); + struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; + struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma; + struct fsl_qdma_comp *comp_temp, *_comp_temp; + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + vchan_get_all_descriptors(&fsl_chan->vchan, &head); + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + + if (!fsl_queue->comp_pool && !fsl_queue->comp_pool) + return; + + list_for_each_entry_safe(comp_temp, _comp_temp, + &fsl_queue->comp_used, list) { + dma_pool_free(fsl_queue->comp_pool, + comp_temp->virt_addr, + comp_temp->bus_addr); + dma_pool_free(fsl_queue->desc_pool, + comp_temp->desc_virt_addr, + comp_temp->desc_bus_addr); + list_del(&comp_temp->list); + kfree(comp_temp); + } + + list_for_each_entry_safe(comp_temp, _comp_temp, + &fsl_queue->comp_free, list) { + dma_pool_free(fsl_queue->comp_pool, + comp_temp->virt_addr, + comp_temp->bus_addr); + dma_pool_free(fsl_queue->desc_pool, + comp_temp->desc_virt_addr, + comp_temp->desc_bus_addr); + list_del(&comp_temp->list); + kfree(comp_temp); + } + + dma_pool_destroy(fsl_queue->comp_pool); + dma_pool_destroy(fsl_queue->desc_pool); + + fsl_qdma->desc_allocated--; + fsl_queue->comp_pool = NULL; + fsl_queue->desc_pool = NULL; +} + +static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp, + dma_addr_t dst, dma_addr_t src, u32 len) +{ + struct fsl_qdma_format *sdf, *ddf; + struct fsl_qdma_format *ccdf, *csgf_desc, *csgf_src, *csgf_dest; + + ccdf = fsl_comp->virt_addr; + csgf_desc = fsl_comp->virt_addr + 1; + csgf_src = fsl_comp->virt_addr + 2; + csgf_dest = fsl_comp->virt_addr + 3; + sdf = fsl_comp->desc_virt_addr; + ddf = fsl_comp->desc_virt_addr + 1; + + memset(fsl_comp->virt_addr, 0, FSL_QDMA_COMMAND_BUFFER_SIZE); + memset(fsl_comp->desc_virt_addr, 0, FSL_QDMA_DESCRIPTOR_BUFFER_SIZE); + /* Head Command Descriptor(Frame Descriptor) */ + qdma_desc_addr_set64(ccdf, fsl_comp->bus_addr + 16); + qdma_ccdf_set_format(ccdf, qdma_ccdf_get_offset(ccdf)); + qdma_ccdf_set_ser(ccdf, qdma_ccdf_get_status(ccdf)); + /* Status notification is enqueued to status queue. */ + /* Compound Command Descriptor(Frame List Table) */ + qdma_desc_addr_set64(csgf_desc, fsl_comp->desc_bus_addr); + /* It must be 32 as Compound S/G Descriptor */ + qdma_csgf_set_len(csgf_desc, 32); + qdma_desc_addr_set64(csgf_src, src); + qdma_csgf_set_len(csgf_src, len); + qdma_desc_addr_set64(csgf_dest, dst); + qdma_csgf_set_len(csgf_dest, len); + /* This entry is the last entry. */ + qdma_csgf_set_f(csgf_dest, len); + /* Descriptor Buffer */ + sdf->data = + cpu_to_le64(FSL_QDMA_CMD_RWTTYPE << + FSL_QDMA_CMD_RWTTYPE_OFFSET); + ddf->data = + cpu_to_le64(FSL_QDMA_CMD_RWTTYPE << + FSL_QDMA_CMD_RWTTYPE_OFFSET); + ddf->data |= + cpu_to_le64(FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET); +} + +/* + * Pre-request full command descriptor for enqueue. + */ +static int fsl_qdma_pre_request_enqueue_desc(struct fsl_qdma_queue *queue) +{ + int i; + struct fsl_qdma_comp *comp_temp, *_comp_temp; + + for (i = 0; i < queue->n_cq + FSL_COMMAND_QUEUE_OVERFLLOW; i++) { + comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL); + if (!comp_temp) + goto err_alloc; + comp_temp->virt_addr = + dma_pool_alloc(queue->comp_pool, GFP_KERNEL, + &comp_temp->bus_addr); + if (!comp_temp->virt_addr) + goto err_dma_alloc; + + comp_temp->desc_virt_addr = + dma_pool_alloc(queue->desc_pool, GFP_KERNEL, + &comp_temp->desc_bus_addr); + if (!comp_temp->desc_virt_addr) + goto err_desc_dma_alloc; + + list_add_tail(&comp_temp->list, &queue->comp_free); + } + + return 0; + +err_desc_dma_alloc: + dma_pool_free(queue->comp_pool, comp_temp->virt_addr, + comp_temp->bus_addr); + +err_dma_alloc: + kfree(comp_temp); + +err_alloc: + list_for_each_entry_safe(comp_temp, _comp_temp, + &queue->comp_free, list) { + if (comp_temp->virt_addr) + dma_pool_free(queue->comp_pool, + comp_temp->virt_addr, + comp_temp->bus_addr); + if (comp_temp->desc_virt_addr) + dma_pool_free(queue->desc_pool, + comp_temp->desc_virt_addr, + comp_temp->desc_bus_addr); + + list_del(&comp_temp->list); + kfree(comp_temp); + } + + return -ENOMEM; +} + +/* + * Request a command descriptor for enqueue. + */ +static struct fsl_qdma_comp +*fsl_qdma_request_enqueue_desc(struct fsl_qdma_chan *fsl_chan) +{ + unsigned long flags; + struct fsl_qdma_comp *comp_temp; + int timeout = FSL_QDMA_COMP_TIMEOUT; + struct fsl_qdma_queue *queue = fsl_chan->queue; + + while (timeout--) { + spin_lock_irqsave(&queue->queue_lock, flags); + if (!list_empty(&queue->comp_free)) { + comp_temp = list_first_entry(&queue->comp_free, + struct fsl_qdma_comp, + list); + list_del(&comp_temp->list); + + spin_unlock_irqrestore(&queue->queue_lock, flags); + comp_temp->qchan = fsl_chan; + return comp_temp; + } + spin_unlock_irqrestore(&queue->queue_lock, flags); + udelay(1); + } + + return NULL; +} + +static struct fsl_qdma_queue +*fsl_qdma_alloc_queue_resources(struct platform_device *pdev, + struct fsl_qdma_engine *fsl_qdma) +{ + int ret, len, i, j; + int queue_num, block_number; + unsigned int queue_size[FSL_QDMA_QUEUE_MAX]; + struct fsl_qdma_queue *queue_head, *queue_temp; + + queue_num = fsl_qdma->n_queues; + block_number = fsl_qdma->block_number; + + if (queue_num > FSL_QDMA_QUEUE_MAX) + queue_num = FSL_QDMA_QUEUE_MAX; + len = sizeof(*queue_head) * queue_num * block_number; + queue_head = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); + if (!queue_head) + return NULL; + + ret = device_property_read_u32_array(&pdev->dev, "queue-sizes", + queue_size, queue_num); + if (ret) { + dev_err(&pdev->dev, "Can't get queue-sizes.\n"); + return NULL; + } + for (j = 0; j < block_number; j++) { + for (i = 0; i < queue_num; i++) { + if (queue_size[i] > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX || + queue_size[i] < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) { + dev_err(&pdev->dev, + "Get wrong queue-sizes.\n"); + return NULL; + } + queue_temp = queue_head + i + (j * queue_num); + + queue_temp->cq = + dma_alloc_coherent(&pdev->dev, + sizeof(struct fsl_qdma_format) * + queue_size[i], + &queue_temp->bus_addr, + GFP_KERNEL); + if (!queue_temp->cq) + return NULL; + queue_temp->block_base = fsl_qdma->block_base + + FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); + queue_temp->n_cq = queue_size[i]; + queue_temp->id = i; + queue_temp->virt_head = queue_temp->cq; + queue_temp->virt_tail = queue_temp->cq; + /* + * List for queue command buffer + */ + INIT_LIST_HEAD(&queue_temp->comp_used); + spin_lock_init(&queue_temp->queue_lock); + } + } + return queue_head; +} + +static struct fsl_qdma_queue +*fsl_qdma_prep_status_queue(struct platform_device *pdev) +{ + int ret; + unsigned int status_size; + struct fsl_qdma_queue *status_head; + struct device_node *np = pdev->dev.of_node; + + ret = of_property_read_u32(np, "status-sizes", &status_size); + if (ret) { + dev_err(&pdev->dev, "Can't get status-sizes.\n"); + return NULL; + } + if (status_size > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX || + status_size < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) { + dev_err(&pdev->dev, "Get wrong status_size.\n"); + return NULL; + } + status_head = devm_kzalloc(&pdev->dev, + sizeof(*status_head), GFP_KERNEL); + if (!status_head) + return NULL; + + /* + * Buffer for queue command + */ + status_head->cq = dma_alloc_coherent(&pdev->dev, + sizeof(struct fsl_qdma_format) * + status_size, + &status_head->bus_addr, + GFP_KERNEL); + if (!status_head->cq) { + devm_kfree(&pdev->dev, status_head); + return NULL; + } + status_head->n_cq = status_size; + status_head->virt_head = status_head->cq; + status_head->virt_tail = status_head->cq; + status_head->comp_pool = NULL; + + return status_head; +} + +static int fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma) +{ + u32 reg; + int i, j, count = FSL_QDMA_HALT_COUNT; + void __iomem *block, *ctrl = fsl_qdma->ctrl_base; + + /* Disable the command queue and wait for idle state. */ + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR); + reg |= FSL_QDMA_DMR_DQD; + qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR); + for (j = 0; j < fsl_qdma->block_number; j++) { + block = fsl_qdma->block_base + + FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); + for (i = 0; i < FSL_QDMA_QUEUE_NUM_MAX; i++) + qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQMR(i)); + } + while (1) { + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DSR); + if (!(reg & FSL_QDMA_DSR_DB)) + break; + if (count-- < 0) + return -EBUSY; + udelay(100); + } + + for (j = 0; j < fsl_qdma->block_number; j++) { + block = fsl_qdma->block_base + + FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); + + /* Disable status queue. */ + qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BSQMR); + + /* + * clear the command queue interrupt detect register for + * all queues. + */ + qdma_writel(fsl_qdma, FSL_QDMA_BCQIDR_CLEAR, + block + FSL_QDMA_BCQIDR(0)); + } + + return 0; +} + +static int +fsl_qdma_queue_transfer_complete(struct fsl_qdma_engine *fsl_qdma, + void *block, + int id) +{ + bool duplicate; + u32 reg, i, count; + struct fsl_qdma_queue *temp_queue; + struct fsl_qdma_format *status_addr; + struct fsl_qdma_comp *fsl_comp = NULL; + struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue; + struct fsl_qdma_queue *fsl_status = fsl_qdma->status[id]; + + count = FSL_QDMA_MAX_SIZE; + + while (count--) { + duplicate = 0; + reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQSR); + if (reg & FSL_QDMA_BSQSR_QE) + return 0; + + status_addr = fsl_status->virt_head; + + if (qdma_ccdf_get_queue(status_addr) == + __this_cpu_read(pre.queue) && + qdma_ccdf_addr_get64(status_addr) == + __this_cpu_read(pre.addr)) + duplicate = 1; + i = qdma_ccdf_get_queue(status_addr) + + id * fsl_qdma->n_queues; + __this_cpu_write(pre.addr, qdma_ccdf_addr_get64(status_addr)); + __this_cpu_write(pre.queue, qdma_ccdf_get_queue(status_addr)); + temp_queue = fsl_queue + i; + + spin_lock(&temp_queue->queue_lock); + if (list_empty(&temp_queue->comp_used)) { + if (!duplicate) { + spin_unlock(&temp_queue->queue_lock); + return -EAGAIN; + } + } else { + fsl_comp = list_first_entry(&temp_queue->comp_used, + struct fsl_qdma_comp, list); + if (fsl_comp->bus_addr + 16 != + __this_cpu_read(pre.addr)) { + if (!duplicate) { + spin_unlock(&temp_queue->queue_lock); + return -EAGAIN; + } + } + } + + if (duplicate) { + reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR); + reg |= FSL_QDMA_BSQMR_DI; + qdma_desc_addr_set64(status_addr, 0x0); + fsl_status->virt_head++; + if (fsl_status->virt_head == fsl_status->cq + + fsl_status->n_cq) + fsl_status->virt_head = fsl_status->cq; + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR); + spin_unlock(&temp_queue->queue_lock); + continue; + } + list_del(&fsl_comp->list); + + reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR); + reg |= FSL_QDMA_BSQMR_DI; + qdma_desc_addr_set64(status_addr, 0x0); + fsl_status->virt_head++; + if (fsl_status->virt_head == fsl_status->cq + fsl_status->n_cq) + fsl_status->virt_head = fsl_status->cq; + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR); + spin_unlock(&temp_queue->queue_lock); + + spin_lock(&fsl_comp->qchan->vchan.lock); + vchan_cookie_complete(&fsl_comp->vdesc); + fsl_comp->qchan->status = DMA_COMPLETE; + spin_unlock(&fsl_comp->qchan->vchan.lock); + } + + return 0; +} + +static irqreturn_t fsl_qdma_error_handler(int irq, void *dev_id) +{ + unsigned int intr; + struct fsl_qdma_engine *fsl_qdma = dev_id; + void __iomem *status = fsl_qdma->status_base; + + intr = qdma_readl(fsl_qdma, status + FSL_QDMA_DEDR); + + if (intr) { + dev_err(fsl_qdma->dma_dev.dev, "DMA transaction error!\n"); + return IRQ_NONE; + } + + qdma_writel(fsl_qdma, FSL_QDMA_DEDR_CLEAR, status + FSL_QDMA_DEDR); + return IRQ_HANDLED; +} + +static irqreturn_t fsl_qdma_queue_handler(int irq, void *dev_id) +{ + int id; + unsigned int intr, reg; + struct fsl_qdma_engine *fsl_qdma = dev_id; + void __iomem *block, *ctrl = fsl_qdma->ctrl_base; + + id = irq - fsl_qdma->irq_base; + if (id < 0 && id > fsl_qdma->block_number) { + dev_err(fsl_qdma->dma_dev.dev, + "irq %d is wrong irq_base is %d\n", + irq, fsl_qdma->irq_base); + } + + block = fsl_qdma->block_base + + FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, id); + + intr = qdma_readl(fsl_qdma, block + FSL_QDMA_BCQIDR(0)); + + if ((intr & FSL_QDMA_CQIDR_SQT) != 0) + intr = fsl_qdma_queue_transfer_complete(fsl_qdma, block, id); + + if (intr != 0) { + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR); + reg |= FSL_QDMA_DMR_DQD; + qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR); + qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQIER(0)); + dev_err(fsl_qdma->dma_dev.dev, "QDMA: status err!\n"); + } + + /* Clear all detected events and interrupts. */ + qdma_writel(fsl_qdma, FSL_QDMA_BCQIDR_CLEAR, + block + FSL_QDMA_BCQIDR(0)); + + return IRQ_HANDLED; +} + +static int +fsl_qdma_irq_init(struct platform_device *pdev, + struct fsl_qdma_engine *fsl_qdma) +{ + int i; + int cpu; + int ret; + char irq_name[20]; + + fsl_qdma->error_irq = + platform_get_irq_byname(pdev, "qdma-error"); + if (fsl_qdma->error_irq < 0) { + dev_err(&pdev->dev, "Can't get qdma controller irq.\n"); + return fsl_qdma->error_irq; + } + + ret = devm_request_irq(&pdev->dev, fsl_qdma->error_irq, + fsl_qdma_error_handler, 0, + "qDMA error", fsl_qdma); + if (ret) { + dev_err(&pdev->dev, "Can't register qDMA controller IRQ.\n"); + return ret; + } + + for (i = 0; i < fsl_qdma->block_number; i++) { + sprintf(irq_name, "qdma-queue%d", i); + fsl_qdma->queue_irq[i] = + platform_get_irq_byname(pdev, irq_name); + + if (fsl_qdma->queue_irq[i] < 0) { + dev_err(&pdev->dev, + "Can't get qdma queue %d irq.\n", i); + return fsl_qdma->queue_irq[i]; + } + + ret = devm_request_irq(&pdev->dev, + fsl_qdma->queue_irq[i], + fsl_qdma_queue_handler, + 0, + "qDMA queue", + fsl_qdma); + if (ret) { + dev_err(&pdev->dev, + "Can't register qDMA queue IRQ.\n"); + return ret; + } + + cpu = i % num_online_cpus(); + ret = irq_set_affinity_hint(fsl_qdma->queue_irq[i], + get_cpu_mask(cpu)); + if (ret) { + dev_err(&pdev->dev, + "Can't set cpu %d affinity to IRQ %d.\n", + cpu, + fsl_qdma->queue_irq[i]); + return ret; + } + } + + return 0; +} + +static void fsl_qdma_irq_exit(struct platform_device *pdev, + struct fsl_qdma_engine *fsl_qdma) +{ + int i; + + devm_free_irq(&pdev->dev, fsl_qdma->error_irq, fsl_qdma); + for (i = 0; i < fsl_qdma->block_number; i++) + devm_free_irq(&pdev->dev, fsl_qdma->queue_irq[i], fsl_qdma); +} + +static int fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma) +{ + u32 reg; + int i, j, ret; + struct fsl_qdma_queue *temp; + void __iomem *status = fsl_qdma->status_base; + void __iomem *block, *ctrl = fsl_qdma->ctrl_base; + struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue; + + /* Try to halt the qDMA engine first. */ + ret = fsl_qdma_halt(fsl_qdma); + if (ret) { + dev_err(fsl_qdma->dma_dev.dev, "DMA halt failed!"); + return ret; + } + + for (i = 0; i < fsl_qdma->block_number; i++) { + /* + * Clear the command queue interrupt detect register for + * all queues. + */ + + block = fsl_qdma->block_base + + FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, i); + qdma_writel(fsl_qdma, FSL_QDMA_BCQIDR_CLEAR, + block + FSL_QDMA_BCQIDR(0)); + } + + for (j = 0; j < fsl_qdma->block_number; j++) { + block = fsl_qdma->block_base + + FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); + for (i = 0; i < fsl_qdma->n_queues; i++) { + temp = fsl_queue + i + (j * fsl_qdma->n_queues); + /* + * Initialize Command Queue registers to + * point to the first + * command descriptor in memory. + * Dequeue Pointer Address Registers + * Enqueue Pointer Address Registers + */ + + qdma_writel(fsl_qdma, temp->bus_addr, + block + FSL_QDMA_BCQDPA_SADDR(i)); + qdma_writel(fsl_qdma, temp->bus_addr, + block + FSL_QDMA_BCQEPA_SADDR(i)); + + /* Initialize the queue mode. */ + reg = FSL_QDMA_BCQMR_EN; + reg |= FSL_QDMA_BCQMR_CD_THLD(ilog2(temp->n_cq) - 4); + reg |= FSL_QDMA_BCQMR_CQ_SIZE(ilog2(temp->n_cq) - 6); + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BCQMR(i)); + } + + /* + * Workaround for erratum: ERR010812. + * We must enable XOFF to avoid the enqueue rejection occurs. + * Setting SQCCMR ENTER_WM to 0x20. + */ + + qdma_writel(fsl_qdma, FSL_QDMA_SQCCMR_ENTER_WM, + block + FSL_QDMA_SQCCMR); + + /* + * Initialize status queue registers to point to the first + * command descriptor in memory. + * Dequeue Pointer Address Registers + * Enqueue Pointer Address Registers + */ + + qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr, + block + FSL_QDMA_SQEPAR); + qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr, + block + FSL_QDMA_SQDPAR); + /* Initialize status queue interrupt. */ + qdma_writel(fsl_qdma, FSL_QDMA_BCQIER_CQTIE, + block + FSL_QDMA_BCQIER(0)); + qdma_writel(fsl_qdma, FSL_QDMA_BSQICR_ICEN | + FSL_QDMA_BSQICR_ICST(5) | 0x8000, + block + FSL_QDMA_BSQICR); + qdma_writel(fsl_qdma, FSL_QDMA_CQIER_MEIE | + FSL_QDMA_CQIER_TEIE, + block + FSL_QDMA_CQIER); + + /* Initialize the status queue mode. */ + reg = FSL_QDMA_BSQMR_EN; + reg |= FSL_QDMA_BSQMR_CQ_SIZE(ilog2 + (fsl_qdma->status[j]->n_cq) - 6); + + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR); + reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR); + } + + /* Initialize controller interrupt register. */ + qdma_writel(fsl_qdma, FSL_QDMA_DEDR_CLEAR, status + FSL_QDMA_DEDR); + qdma_writel(fsl_qdma, FSL_QDMA_DEIER_CLEAR, status + FSL_QDMA_DEIER); + + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR); + reg &= ~FSL_QDMA_DMR_DQD; + qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR); + + return 0; +} + +static struct dma_async_tx_descriptor * +fsl_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct fsl_qdma_comp *fsl_comp; + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); + + fsl_comp = fsl_qdma_request_enqueue_desc(fsl_chan); + + if (!fsl_comp) + return NULL; + + fsl_qdma_comp_fill_memcpy(fsl_comp, dst, src, len); + + return vchan_tx_prep(&fsl_chan->vchan, &fsl_comp->vdesc, flags); +} + +static void fsl_qdma_enqueue_desc(struct fsl_qdma_chan *fsl_chan) +{ + u32 reg; + struct virt_dma_desc *vdesc; + struct fsl_qdma_comp *fsl_comp; + struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; + void __iomem *block = fsl_queue->block_base; + + reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQSR(fsl_queue->id)); + if (reg & (FSL_QDMA_BCQSR_QF | FSL_QDMA_BCQSR_XOFF)) + return; + vdesc = vchan_next_desc(&fsl_chan->vchan); + if (!vdesc) + return; + list_del(&vdesc->node); + fsl_comp = to_fsl_qdma_comp(vdesc); + + memcpy(fsl_queue->virt_head++, + fsl_comp->virt_addr, sizeof(struct fsl_qdma_format)); + if (fsl_queue->virt_head == fsl_queue->cq + fsl_queue->n_cq) + fsl_queue->virt_head = fsl_queue->cq; + + list_add_tail(&fsl_comp->list, &fsl_queue->comp_used); + barrier(); + reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQMR(fsl_queue->id)); + reg |= FSL_QDMA_BCQMR_EI; + qdma_writel(fsl_chan->qdma, reg, block + FSL_QDMA_BCQMR(fsl_queue->id)); + fsl_chan->status = DMA_IN_PROGRESS; +} + +static void fsl_qdma_free_desc(struct virt_dma_desc *vdesc) +{ + unsigned long flags; + struct fsl_qdma_comp *fsl_comp; + struct fsl_qdma_queue *fsl_queue; + + fsl_comp = to_fsl_qdma_comp(vdesc); + fsl_queue = fsl_comp->qchan->queue; + + spin_lock_irqsave(&fsl_queue->queue_lock, flags); + list_add_tail(&fsl_comp->list, &fsl_queue->comp_free); + spin_unlock_irqrestore(&fsl_queue->queue_lock, flags); +} + +static void fsl_qdma_issue_pending(struct dma_chan *chan) +{ + unsigned long flags; + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); + struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; + + spin_lock_irqsave(&fsl_queue->queue_lock, flags); + spin_lock(&fsl_chan->vchan.lock); + if (vchan_issue_pending(&fsl_chan->vchan)) + fsl_qdma_enqueue_desc(fsl_chan); + spin_unlock(&fsl_chan->vchan.lock); + spin_unlock_irqrestore(&fsl_queue->queue_lock, flags); +} + +static void fsl_qdma_synchronize(struct dma_chan *chan) +{ + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); + + vchan_synchronize(&fsl_chan->vchan); +} + +static int fsl_qdma_terminate_all(struct dma_chan *chan) +{ + LIST_HEAD(head); + unsigned long flags; + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + vchan_get_all_descriptors(&fsl_chan->vchan, &head); + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + return 0; +} + +static int fsl_qdma_alloc_chan_resources(struct dma_chan *chan) +{ + int ret; + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); + struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma; + struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; + + if (fsl_queue->comp_pool && fsl_queue->desc_pool) + return fsl_qdma->desc_allocated; + + INIT_LIST_HEAD(&fsl_queue->comp_free); + + /* + * The dma pool for queue command buffer + */ + fsl_queue->comp_pool = + dma_pool_create("comp_pool", + chan->device->dev, + FSL_QDMA_COMMAND_BUFFER_SIZE, + 64, 0); + if (!fsl_queue->comp_pool) + return -ENOMEM; + + /* + * The dma pool for Descriptor(SD/DD) buffer + */ + fsl_queue->desc_pool = + dma_pool_create("desc_pool", + chan->device->dev, + FSL_QDMA_DESCRIPTOR_BUFFER_SIZE, + 32, 0); + if (!fsl_queue->desc_pool) + goto err_desc_pool; + + ret = fsl_qdma_pre_request_enqueue_desc(fsl_queue); + if (ret) { + dev_err(chan->device->dev, + "failed to alloc dma buffer for S/G descriptor\n"); + goto err_mem; + } + + fsl_qdma->desc_allocated++; + return fsl_qdma->desc_allocated; + +err_mem: + dma_pool_destroy(fsl_queue->desc_pool); +err_desc_pool: + dma_pool_destroy(fsl_queue->comp_pool); + return -ENOMEM; +} + +static int fsl_qdma_probe(struct platform_device *pdev) +{ + int ret, i; + int blk_num, blk_off; + u32 len, chans, queues; + struct resource *res; + struct fsl_qdma_chan *fsl_chan; + struct fsl_qdma_engine *fsl_qdma; + struct device_node *np = pdev->dev.of_node; + + ret = of_property_read_u32(np, "dma-channels", &chans); + if (ret) { + dev_err(&pdev->dev, "Can't get dma-channels.\n"); + return ret; + } + + ret = of_property_read_u32(np, "block-offset", &blk_off); + if (ret) { + dev_err(&pdev->dev, "Can't get block-offset.\n"); + return ret; + } + + ret = of_property_read_u32(np, "block-number", &blk_num); + if (ret) { + dev_err(&pdev->dev, "Can't get block-number.\n"); + return ret; + } + + blk_num = min_t(int, blk_num, num_online_cpus()); + + len = sizeof(*fsl_qdma); + fsl_qdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); + if (!fsl_qdma) + return -ENOMEM; + + len = sizeof(*fsl_chan) * chans; + fsl_qdma->chans = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); + if (!fsl_qdma->chans) + return -ENOMEM; + + len = sizeof(struct fsl_qdma_queue *) * blk_num; + fsl_qdma->status = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); + if (!fsl_qdma->status) + return -ENOMEM; + + len = sizeof(int) * blk_num; + fsl_qdma->queue_irq = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); + if (!fsl_qdma->queue_irq) + return -ENOMEM; + + ret = of_property_read_u32(np, "fsl,dma-queues", &queues); + if (ret) { + dev_err(&pdev->dev, "Can't get queues.\n"); + return ret; + } + + fsl_qdma->desc_allocated = 0; + fsl_qdma->n_chans = chans; + fsl_qdma->n_queues = queues; + fsl_qdma->block_number = blk_num; + fsl_qdma->block_offset = blk_off; + + mutex_init(&fsl_qdma->fsl_qdma_mutex); + + for (i = 0; i < fsl_qdma->block_number; i++) { + fsl_qdma->status[i] = fsl_qdma_prep_status_queue(pdev); + if (!fsl_qdma->status[i]) + return -ENOMEM; + } + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + fsl_qdma->ctrl_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(fsl_qdma->ctrl_base)) + return PTR_ERR(fsl_qdma->ctrl_base); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + fsl_qdma->status_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(fsl_qdma->status_base)) + return PTR_ERR(fsl_qdma->status_base); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 2); + fsl_qdma->block_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(fsl_qdma->block_base)) + return PTR_ERR(fsl_qdma->block_base); + fsl_qdma->queue = fsl_qdma_alloc_queue_resources(pdev, fsl_qdma); + if (!fsl_qdma->queue) + return -ENOMEM; + + ret = fsl_qdma_irq_init(pdev, fsl_qdma); + if (ret) + return ret; + + fsl_qdma->irq_base = platform_get_irq_byname(pdev, "qdma-queue0"); + fsl_qdma->feature = of_property_read_bool(np, "big-endian"); + INIT_LIST_HEAD(&fsl_qdma->dma_dev.channels); + + for (i = 0; i < fsl_qdma->n_chans; i++) { + struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i]; + + fsl_chan->qdma = fsl_qdma; + fsl_chan->queue = fsl_qdma->queue + i % (fsl_qdma->n_queues * + fsl_qdma->block_number); + fsl_chan->vchan.desc_free = fsl_qdma_free_desc; + vchan_init(&fsl_chan->vchan, &fsl_qdma->dma_dev); + } + + dma_cap_set(DMA_MEMCPY, fsl_qdma->dma_dev.cap_mask); + + fsl_qdma->dma_dev.dev = &pdev->dev; + fsl_qdma->dma_dev.device_free_chan_resources = + fsl_qdma_free_chan_resources; + fsl_qdma->dma_dev.device_alloc_chan_resources = + fsl_qdma_alloc_chan_resources; + fsl_qdma->dma_dev.device_tx_status = dma_cookie_status; + fsl_qdma->dma_dev.device_prep_dma_memcpy = fsl_qdma_prep_memcpy; + fsl_qdma->dma_dev.device_issue_pending = fsl_qdma_issue_pending; + fsl_qdma->dma_dev.device_synchronize = fsl_qdma_synchronize; + fsl_qdma->dma_dev.device_terminate_all = fsl_qdma_terminate_all; + + dma_set_mask(&pdev->dev, DMA_BIT_MASK(40)); + + platform_set_drvdata(pdev, fsl_qdma); + + ret = dma_async_device_register(&fsl_qdma->dma_dev); + if (ret) { + dev_err(&pdev->dev, + "Can't register NXP Layerscape qDMA engine.\n"); + return ret; + } + + ret = fsl_qdma_reg_init(fsl_qdma); + if (ret) { + dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n"); + return ret; + } + + return 0; +} + +static void fsl_qdma_cleanup_vchan(struct dma_device *dmadev) +{ + struct fsl_qdma_chan *chan, *_chan; + + list_for_each_entry_safe(chan, _chan, + &dmadev->channels, vchan.chan.device_node) { + list_del(&chan->vchan.chan.device_node); + tasklet_kill(&chan->vchan.task); + } +} + +static int fsl_qdma_remove(struct platform_device *pdev) +{ + int i; + struct fsl_qdma_queue *status; + struct device_node *np = pdev->dev.of_node; + struct fsl_qdma_engine *fsl_qdma = platform_get_drvdata(pdev); + + fsl_qdma_irq_exit(pdev, fsl_qdma); + fsl_qdma_cleanup_vchan(&fsl_qdma->dma_dev); + of_dma_controller_free(np); + dma_async_device_unregister(&fsl_qdma->dma_dev); + + for (i = 0; i < fsl_qdma->block_number; i++) { + status = fsl_qdma->status[i]; + dma_free_coherent(&pdev->dev, sizeof(struct fsl_qdma_format) * + status->n_cq, status->cq, status->bus_addr); + } + return 0; +} + +static const struct of_device_id fsl_qdma_dt_ids[] = { + { .compatible = "fsl,ls1021a-qdma", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, fsl_qdma_dt_ids); + +static struct platform_driver fsl_qdma_driver = { + .driver = { + .name = "fsl-qdma", + .of_match_table = fsl_qdma_dt_ids, + }, + .probe = fsl_qdma_probe, + .remove = fsl_qdma_remove, +}; + +module_platform_driver(fsl_qdma_driver); + +MODULE_ALIAS("platform:fsl-qdma"); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("NXP Layerscape qDMA engine driver"); diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index 9d360a3fbae31..1e38e6b940062 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -53,42 +53,42 @@ static const char msg_ld_oom[] = "No free memory for link descriptor"; static void set_sr(struct fsldma_chan *chan, u32 val) { - DMA_OUT(chan, &chan->regs->sr, val, 32); + FSL_DMA_OUT(chan, &chan->regs->sr, val, 32); } static u32 get_sr(struct fsldma_chan *chan) { - return DMA_IN(chan, &chan->regs->sr, 32); + return FSL_DMA_IN(chan, &chan->regs->sr, 32); } static void set_mr(struct fsldma_chan *chan, u32 val) { - DMA_OUT(chan, &chan->regs->mr, val, 32); + FSL_DMA_OUT(chan, &chan->regs->mr, val, 32); } static u32 get_mr(struct fsldma_chan *chan) { - return DMA_IN(chan, &chan->regs->mr, 32); + return FSL_DMA_IN(chan, &chan->regs->mr, 32); } static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr) { - DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64); + FSL_DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64); } static dma_addr_t get_cdar(struct fsldma_chan *chan) { - return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN; + return FSL_DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN; } static void set_bcr(struct fsldma_chan *chan, u32 val) { - DMA_OUT(chan, &chan->regs->bcr, val, 32); + FSL_DMA_OUT(chan, &chan->regs->bcr, val, 32); } static u32 get_bcr(struct fsldma_chan *chan) { - return DMA_IN(chan, &chan->regs->bcr, 32); + return FSL_DMA_IN(chan, &chan->regs->bcr, 32); } /* diff --git a/drivers/dma/fsldma.h b/drivers/dma/fsldma.h index 4787d485dd762..a9b12f82b5c35 100644 --- a/drivers/dma/fsldma.h +++ b/drivers/dma/fsldma.h @@ -196,39 +196,67 @@ struct fsldma_chan { #define to_fsl_desc(lh) container_of(lh, struct fsl_desc_sw, node) #define tx_to_fsl_desc(tx) container_of(tx, struct fsl_desc_sw, async_tx) -#ifndef __powerpc64__ -static u64 in_be64(const u64 __iomem *addr) +#ifdef CONFIG_PPC +#define fsl_ioread32(p) in_le32(p) +#define fsl_ioread32be(p) in_be32(p) +#define fsl_iowrite32(v, p) out_le32(p, v) +#define fsl_iowrite32be(v, p) out_be32(p, v) + +#ifdef __powerpc64__ +#define fsl_ioread64(p) in_le64(p) +#define fsl_ioread64be(p) in_be64(p) +#define fsl_iowrite64(v, p) out_le64(p, v) +#define fsl_iowrite64be(v, p) out_be64(p, v) +#else +static u64 fsl_ioread64(const u64 __iomem *addr) { - return ((u64)in_be32((u32 __iomem *)addr) << 32) | - (in_be32((u32 __iomem *)addr + 1)); + u32 fsl_addr = lower_32_bits(addr); + u64 fsl_addr_hi = (u64)in_le32((u32 *)(fsl_addr + 1)) << 32; + + return fsl_addr_hi | in_le32((u32 *)fsl_addr); } -static void out_be64(u64 __iomem *addr, u64 val) +static void fsl_iowrite64(u64 val, u64 __iomem *addr) { - out_be32((u32 __iomem *)addr, val >> 32); - out_be32((u32 __iomem *)addr + 1, (u32)val); + out_le32((u32 __iomem *)addr + 1, val >> 32); + out_le32((u32 __iomem *)addr, (u32)val); } -/* There is no asm instructions for 64 bits reverse loads and stores */ -static u64 in_le64(const u64 __iomem *addr) +static u64 fsl_ioread64be(const u64 __iomem *addr) { - return ((u64)in_le32((u32 __iomem *)addr + 1) << 32) | - (in_le32((u32 __iomem *)addr)); + u32 fsl_addr = lower_32_bits(addr); + u64 fsl_addr_hi = (u64)in_be32((u32 *)fsl_addr) << 32; + + return fsl_addr_hi | in_be32((u32 *)(fsl_addr + 1)); } -static void out_le64(u64 __iomem *addr, u64 val) +static void fsl_iowrite64be(u64 val, u64 __iomem *addr) { - out_le32((u32 __iomem *)addr + 1, val >> 32); - out_le32((u32 __iomem *)addr, (u32)val); + out_be32((u32 __iomem *)addr, val >> 32); + out_be32((u32 __iomem *)addr + 1, (u32)val); } #endif +#endif -#define DMA_IN(fsl_chan, addr, width) \ - (((fsl_chan)->feature & FSL_DMA_BIG_ENDIAN) ? \ - in_be##width(addr) : in_le##width(addr)) -#define DMA_OUT(fsl_chan, addr, val, width) \ - (((fsl_chan)->feature & FSL_DMA_BIG_ENDIAN) ? \ - out_be##width(addr, val) : out_le##width(addr, val)) +#if defined(CONFIG_ARM64) || defined(CONFIG_ARM) +#define fsl_ioread32(p) ioread32(p) +#define fsl_ioread32be(p) ioread32be(p) +#define fsl_iowrite32(v, p) iowrite32(v, p) +#define fsl_iowrite32be(v, p) iowrite32be(v, p) +#define fsl_ioread64(p) ioread64(p) +#define fsl_ioread64be(p) ioread64be(p) +#define fsl_iowrite64(v, p) iowrite64(v, p) +#define fsl_iowrite64be(v, p) iowrite64be(v, p) +#endif + +#define FSL_DMA_IN(fsl_dma, addr, width) \ + (((fsl_dma)->feature & FSL_DMA_BIG_ENDIAN) ? \ + fsl_ioread##width##be(addr) : fsl_ioread##width(addr)) + +#define FSL_DMA_OUT(fsl_dma, addr, val, width) \ + (((fsl_dma)->feature & FSL_DMA_BIG_ENDIAN) ? \ + fsl_iowrite##width##be(val, addr) : fsl_iowrite \ + ##width(val, addr)) #define DMA_TO_CPU(fsl_chan, d, width) \ (((fsl_chan)->feature & FSL_DMA_BIG_ENDIAN) ? \ diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c index 4a09af3cd546a..00a089e24150a 100644 --- a/drivers/dma/imx-dma.c +++ b/drivers/dma/imx-dma.c @@ -278,14 +278,14 @@ static int imxdma_hw_chain(struct imxdma_channel *imxdmac) /* * imxdma_sg_next - prepare next chunk for scatter-gather DMA emulation */ -static inline int imxdma_sg_next(struct imxdma_desc *d) +static inline void imxdma_sg_next(struct imxdma_desc *d) { struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan); struct imxdma_engine *imxdma = imxdmac->imxdma; struct scatterlist *sg = d->sg; - unsigned long now; + size_t now; - now = min(d->len, sg_dma_len(sg)); + now = min_t(size_t, d->len, sg_dma_len(sg)); if (d->len != IMX_DMA_LENGTH_LOOP) d->len -= now; @@ -303,8 +303,6 @@ static inline int imxdma_sg_next(struct imxdma_desc *d) imx_dmav1_readl(imxdma, DMA_DAR(imxdmac->channel)), imx_dmav1_readl(imxdma, DMA_SAR(imxdmac->channel)), imx_dmav1_readl(imxdma, DMA_CNTR(imxdmac->channel))); - - return now; } static void imxdma_enable_hw(struct imxdma_desc *d) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 86708fb9bda1f..5f3c1378b90eb 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -377,6 +377,7 @@ struct sdma_channel { unsigned long watermark_level; u32 shp_addr, per_addr; enum dma_status status; + bool context_loaded; struct imx_dma_data data; struct work_struct terminate_worker; }; @@ -440,6 +441,8 @@ struct sdma_engine { unsigned int irq; dma_addr_t bd0_phys; struct sdma_buffer_descriptor *bd0; + /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/ + bool clk_ratio; }; static int sdma_config_write(struct dma_chan *chan, @@ -662,8 +665,11 @@ static int sdma_run_channel0(struct sdma_engine *sdma) dev_err(sdma->dev, "Timeout waiting for CH0 ready\n"); /* Set bits of CONFIG register with dynamic context switching */ - if (readl(sdma->regs + SDMA_H_CONFIG) == 0) - writel_relaxed(SDMA_H_CONFIG_CSM, sdma->regs + SDMA_H_CONFIG); + reg = readl(sdma->regs + SDMA_H_CONFIG); + if ((reg & SDMA_H_CONFIG_CSM) == 0) { + reg |= SDMA_H_CONFIG_CSM; + writel_relaxed(reg, sdma->regs + SDMA_H_CONFIG); + } return ret; } @@ -677,7 +683,7 @@ static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, int ret; unsigned long flags; - buf_virt = dma_alloc_coherent(NULL, size, &buf_phys, GFP_KERNEL); + buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL); if (!buf_virt) { return -ENOMEM; } @@ -696,7 +702,7 @@ static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, spin_unlock_irqrestore(&sdma->channel_0_lock, flags); - dma_free_coherent(NULL, size, buf_virt, buf_phys); + dma_free_coherent(sdma->dev, size, buf_virt, buf_phys); return ret; } @@ -970,6 +976,9 @@ static int sdma_load_context(struct sdma_channel *sdmac) int ret; unsigned long flags; + if (sdmac->context_loaded) + return 0; + if (sdmac->direction == DMA_DEV_TO_MEM) load_address = sdmac->pc_from_device; else if (sdmac->direction == DMA_DEV_TO_DEV) @@ -1012,6 +1021,8 @@ static int sdma_load_context(struct sdma_channel *sdmac) spin_unlock_irqrestore(&sdma->channel_0_lock, flags); + sdmac->context_loaded = true; + return ret; } @@ -1051,6 +1062,7 @@ static void sdma_channel_terminate_work(struct work_struct *work) sdmac->desc = NULL; spin_unlock_irqrestore(&sdmac->vc.lock, flags); vchan_dma_desc_free_list(&sdmac->vc, &head); + sdmac->context_loaded = false; } static int sdma_disable_channel_async(struct dma_chan *chan) @@ -1182,8 +1194,8 @@ static int sdma_request_channel0(struct sdma_engine *sdma) { int ret = -EBUSY; - sdma->bd0 = dma_alloc_coherent(NULL, PAGE_SIZE, &sdma->bd0_phys, - GFP_NOWAIT); + sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys, + GFP_NOWAIT); if (!sdma->bd0) { ret = -ENOMEM; goto out; @@ -1205,8 +1217,8 @@ static int sdma_alloc_bd(struct sdma_desc *desc) u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); int ret = 0; - desc->bd = dma_alloc_coherent(NULL, bd_size, &desc->bd_phys, - GFP_NOWAIT); + desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size, + &desc->bd_phys, GFP_NOWAIT); if (!desc->bd) { ret = -ENOMEM; goto out; @@ -1219,7 +1231,8 @@ static void sdma_free_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); - dma_free_coherent(NULL, bd_size, desc->bd, desc->bd_phys); + dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, + desc->bd_phys); } static void sdma_desc_free(struct virt_dma_desc *vd) @@ -1839,10 +1852,13 @@ static int sdma_init(struct sdma_engine *sdma) if (ret) goto disable_clk_ipg; + if (clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg)) + sdma->clk_ratio = 1; + /* Be sure SDMA has not started yet */ writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); - sdma->channel_control = dma_alloc_coherent(NULL, + sdma->channel_control = dma_alloc_coherent(sdma->dev, MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) + sizeof(struct sdma_context_data), &ccb_phys, GFP_KERNEL); @@ -1879,8 +1895,10 @@ static int sdma_init(struct sdma_engine *sdma) writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR); /* Set bits of CONFIG register but with static context switching */ - /* FIXME: Check whether to set ACR bit depending on clock ratios */ - writel_relaxed(0, sdma->regs + SDMA_H_CONFIG); + if (sdma->clk_ratio) + writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG); + else + writel_relaxed(0, sdma->regs + SDMA_H_CONFIG); writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR); @@ -1903,11 +1921,16 @@ disable_clk_ipg: static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param) { struct sdma_channel *sdmac = to_sdma_chan(chan); + struct sdma_engine *sdma = sdmac->sdma; struct imx_dma_data *data = fn_param; if (!imx_dma_is_general_purpose(chan)) return false; + /* return false if it's not the right device */ + if (sdma->dev->of_node != data->of_node) + return false; + sdmac->data = *data; chan->private = &sdmac->data; @@ -1935,6 +1958,7 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, * be set to sdmac->event_id1. */ data.dma_request2 = 0; + data.of_node = ofdma->of_node; return dma_request_channel(mask, sdma_filter_fn, &data); } @@ -2097,6 +2121,7 @@ static int sdma_probe(struct platform_device *pdev) sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy; sdma->dma_device.device_issue_pending = sdma_issue_pending; sdma->dma_device.dev->dma_parms = &sdma->dma_parms; + sdma->dma_device.copy_align = 2; dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT); platform_set_drvdata(pdev, sdma); diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 23fb2fa040002..f373a139e0c37 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -372,6 +372,7 @@ struct ioat_ring_ent ** ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags) { struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; struct ioat_ring_ent **ring; int total_descs = 1 << order; int i, chunks; @@ -437,6 +438,17 @@ ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags) } ring[i]->hw->next = ring[0]->txd.phys; + /* setup descriptor pre-fetching for v3.4 */ + if (ioat_dma->cap & IOAT_CAP_DPS) { + u16 drsctl = IOAT_CHAN_DRSZ_2MB | IOAT_CHAN_DRS_EN; + + if (chunks == 1) + drsctl |= IOAT_CHAN_DRS_AUTOWRAP; + + writew(drsctl, ioat_chan->reg_base + IOAT_CHAN_DRSCTL_OFFSET); + + } + return ring; } diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 1ab42ec2b7ff1..aaafd0e882b5d 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -27,7 +27,7 @@ #include "registers.h" #include "hw.h" -#define IOAT_DMA_VERSION "4.00" +#define IOAT_DMA_VERSION "5.00" #define IOAT_DMA_DCA_ANY_CPU ~0 diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h index abcc51b343cec..781c94de8e810 100644 --- a/drivers/dma/ioat/hw.h +++ b/drivers/dma/ioat/hw.h @@ -66,11 +66,14 @@ #define PCI_DEVICE_ID_INTEL_IOAT_SKX 0x2021 +#define PCI_DEVICE_ID_INTEL_IOAT_ICX 0x0b00 + #define IOAT_VER_1_2 0x12 /* Version 1.2 */ #define IOAT_VER_2_0 0x20 /* Version 2.0 */ #define IOAT_VER_3_0 0x30 /* Version 3.0 */ #define IOAT_VER_3_2 0x32 /* Version 3.2 */ #define IOAT_VER_3_3 0x33 /* Version 3.3 */ +#define IOAT_VER_3_4 0x34 /* Version 3.4 */ int system_has_dca_enabled(struct pci_dev *pdev); diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c index 2d810dfcdc484..d41dc9a9ff68d 100644 --- a/drivers/dma/ioat/init.c +++ b/drivers/dma/ioat/init.c @@ -119,6 +119,9 @@ static const struct pci_device_id ioat_pci_tbl[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3) }, + /* I/OAT v3.4 platforms */ + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_ICX) }, + { 0, } }; MODULE_DEVICE_TABLE(pci, ioat_pci_tbl); @@ -135,10 +138,10 @@ static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma); static int ioat_dca_enabled = 1; module_param(ioat_dca_enabled, int, 0644); MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); -int ioat_pending_level = 4; +int ioat_pending_level = 7; module_param(ioat_pending_level, int, 0644); MODULE_PARM_DESC(ioat_pending_level, - "high-water mark for pushing ioat descriptors (default: 4)"); + "high-water mark for pushing ioat descriptors (default: 7)"); static char ioat_interrupt_style[32] = "msix"; module_param_string(ioat_interrupt_style, ioat_interrupt_style, sizeof(ioat_interrupt_style), 0644); @@ -635,6 +638,11 @@ static void ioat_free_chan_resources(struct dma_chan *c) ioat_stop(ioat_chan); ioat_reset_hw(ioat_chan); + /* Put LTR to idle */ + if (ioat_dma->version >= IOAT_VER_3_4) + writeb(IOAT_CHAN_LTR_SWSEL_IDLE, + ioat_chan->reg_base + IOAT_CHAN_LTR_SWSEL_OFFSET); + spin_lock_bh(&ioat_chan->cleanup_lock); spin_lock_bh(&ioat_chan->prep_lock); descs = ioat_ring_space(ioat_chan); @@ -724,6 +732,28 @@ static int ioat_alloc_chan_resources(struct dma_chan *c) spin_unlock_bh(&ioat_chan->prep_lock); spin_unlock_bh(&ioat_chan->cleanup_lock); + /* Setting up LTR values for 3.4 or later */ + if (ioat_chan->ioat_dma->version >= IOAT_VER_3_4) { + u32 lat_val; + + lat_val = IOAT_CHAN_LTR_ACTIVE_SNVAL | + IOAT_CHAN_LTR_ACTIVE_SNLATSCALE | + IOAT_CHAN_LTR_ACTIVE_SNREQMNT; + writel(lat_val, ioat_chan->reg_base + + IOAT_CHAN_LTR_ACTIVE_OFFSET); + + lat_val = IOAT_CHAN_LTR_IDLE_SNVAL | + IOAT_CHAN_LTR_IDLE_SNLATSCALE | + IOAT_CHAN_LTR_IDLE_SNREQMNT; + writel(lat_val, ioat_chan->reg_base + + IOAT_CHAN_LTR_IDLE_OFFSET); + + /* Select to active */ + writeb(IOAT_CHAN_LTR_SWSEL_ACTIVE, + ioat_chan->reg_base + + IOAT_CHAN_LTR_SWSEL_OFFSET); + } + ioat_start_null_desc(ioat_chan); /* check that we got off the ground */ @@ -1185,6 +1215,10 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) if (err) return err; + if (ioat_dma->cap & IOAT_CAP_DPS) + writeb(ioat_pending_level + 1, + ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET); + return 0; } @@ -1350,6 +1384,8 @@ static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_set_drvdata(pdev, device); device->version = readb(device->reg_base + IOAT_VER_OFFSET); + if (device->version >= IOAT_VER_3_4) + ioat_dca_enabled = 0; if (device->version >= IOAT_VER_3_0) { if (is_skx_ioat(pdev)) device->version = IOAT_VER_3_2; diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h index 2f3bbc88ff2a0..99c1c24d465da 100644 --- a/drivers/dma/ioat/registers.h +++ b/drivers/dma/ioat/registers.h @@ -84,6 +84,9 @@ #define IOAT_CAP_PQ 0x00000200 #define IOAT_CAP_DWBES 0x00002000 #define IOAT_CAP_RAID16SS 0x00020000 +#define IOAT_CAP_DPS 0x00800000 + +#define IOAT_PREFETCH_LIMIT_OFFSET 0x4C /* CHWPREFLMT */ #define IOAT_CHANNEL_MMIO_SIZE 0x80 /* Each Channel MMIO space is this size */ @@ -243,4 +246,25 @@ #define IOAT_CHANERR_MASK_OFFSET 0x2C /* 32-bit Channel Error Register */ +#define IOAT_CHAN_DRSCTL_OFFSET 0xB6 +#define IOAT_CHAN_DRSZ_4KB 0x0000 +#define IOAT_CHAN_DRSZ_8KB 0x0001 +#define IOAT_CHAN_DRSZ_2MB 0x0009 +#define IOAT_CHAN_DRS_EN 0x0100 +#define IOAT_CHAN_DRS_AUTOWRAP 0x0200 + +#define IOAT_CHAN_LTR_SWSEL_OFFSET 0xBC +#define IOAT_CHAN_LTR_SWSEL_ACTIVE 0x0 +#define IOAT_CHAN_LTR_SWSEL_IDLE 0x1 + +#define IOAT_CHAN_LTR_ACTIVE_OFFSET 0xC0 +#define IOAT_CHAN_LTR_ACTIVE_SNVAL 0x0000 /* 0 us */ +#define IOAT_CHAN_LTR_ACTIVE_SNLATSCALE 0x0800 /* 1us scale */ +#define IOAT_CHAN_LTR_ACTIVE_SNREQMNT 0x8000 /* snoop req enable */ + +#define IOAT_CHAN_LTR_IDLE_OFFSET 0xC4 +#define IOAT_CHAN_LTR_IDLE_SNVAL 0x0258 /* 600 us */ +#define IOAT_CHAN_LTR_IDLE_SNLATSCALE 0x0800 /* 1us scale */ +#define IOAT_CHAN_LTR_IDLE_SNREQMNT 0x8000 /* snoop req enable */ + #endif /* _IOAT_REGISTERS_H_ */ diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c index fdec2b6cfbb0f..5737d92eaeebc 100644 --- a/drivers/dma/k3dma.c +++ b/drivers/dma/k3dma.c @@ -52,8 +52,6 @@ #define CX_SRC 0x814 #define CX_DST 0x818 #define CX_CFG 0x81c -#define AXI_CFG 0x820 -#define AXI_CFG_DEFAULT 0x201201 #define CX_LLI_CHAIN_EN 0x2 #define CX_CFG_EN 0x1 @@ -113,9 +111,18 @@ struct k3_dma_dev { struct dma_pool *pool; u32 dma_channels; u32 dma_requests; + u32 dma_channel_mask; unsigned int irq; }; + +#define K3_FLAG_NOCLK BIT(1) + +struct k3dma_soc_data { + unsigned long flags; +}; + + #define to_k3_dma(dmadev) container_of(dmadev, struct k3_dma_dev, slave) static int k3_dma_config_write(struct dma_chan *chan, @@ -161,7 +168,6 @@ static void k3_dma_set_desc(struct k3_dma_phy *phy, struct k3_desc_hw *hw) writel_relaxed(hw->count, phy->base + CX_CNT0); writel_relaxed(hw->saddr, phy->base + CX_SRC); writel_relaxed(hw->daddr, phy->base + CX_DST); - writel_relaxed(AXI_CFG_DEFAULT, phy->base + AXI_CFG); writel_relaxed(hw->config, phy->base + CX_CFG); } @@ -314,6 +320,9 @@ static void k3_dma_tasklet(unsigned long arg) /* check new channel request in d->chan_pending */ spin_lock_irq(&d->lock); for (pch = 0; pch < d->dma_channels; pch++) { + if (!(d->dma_channel_mask & (1 << pch))) + continue; + p = &d->phy[pch]; if (p->vchan == NULL && !list_empty(&d->chan_pending)) { @@ -331,6 +340,9 @@ static void k3_dma_tasklet(unsigned long arg) spin_unlock_irq(&d->lock); for (pch = 0; pch < d->dma_channels; pch++) { + if (!(d->dma_channel_mask & (1 << pch))) + continue; + if (pch_alloc & (1 << pch)) { p = &d->phy[pch]; c = p->vchan; @@ -790,8 +802,21 @@ static int k3_dma_transfer_resume(struct dma_chan *chan) return 0; } +static const struct k3dma_soc_data k3_v1_dma_data = { + .flags = 0, +}; + +static const struct k3dma_soc_data asp_v1_dma_data = { + .flags = K3_FLAG_NOCLK, +}; + static const struct of_device_id k3_pdma_dt_ids[] = { - { .compatible = "hisilicon,k3-dma-1.0", }, + { .compatible = "hisilicon,k3-dma-1.0", + .data = &k3_v1_dma_data + }, + { .compatible = "hisilicon,hisi-pcm-asp-dma-1.0", + .data = &asp_v1_dma_data + }, {} }; MODULE_DEVICE_TABLE(of, k3_pdma_dt_ids); @@ -810,6 +835,7 @@ static struct dma_chan *k3_of_dma_simple_xlate(struct of_phandle_args *dma_spec, static int k3_dma_probe(struct platform_device *op) { + const struct k3dma_soc_data *soc_data; struct k3_dma_dev *d; const struct of_device_id *of_id; struct resource *iores; @@ -823,6 +849,10 @@ static int k3_dma_probe(struct platform_device *op) if (!d) return -ENOMEM; + soc_data = device_get_match_data(&op->dev); + if (!soc_data) + return -EINVAL; + d->base = devm_ioremap_resource(&op->dev, iores); if (IS_ERR(d->base)) return PTR_ERR(d->base); @@ -833,12 +863,21 @@ static int k3_dma_probe(struct platform_device *op) "dma-channels", &d->dma_channels); of_property_read_u32((&op->dev)->of_node, "dma-requests", &d->dma_requests); + ret = of_property_read_u32((&op->dev)->of_node, + "dma-channel-mask", &d->dma_channel_mask); + if (ret) { + dev_warn(&op->dev, + "dma-channel-mask doesn't exist, considering all as available.\n"); + d->dma_channel_mask = (u32)~0UL; + } } - d->clk = devm_clk_get(&op->dev, NULL); - if (IS_ERR(d->clk)) { - dev_err(&op->dev, "no dma clk\n"); - return PTR_ERR(d->clk); + if (!(soc_data->flags & K3_FLAG_NOCLK)) { + d->clk = devm_clk_get(&op->dev, NULL); + if (IS_ERR(d->clk)) { + dev_err(&op->dev, "no dma clk\n"); + return PTR_ERR(d->clk); + } } irq = platform_get_irq(op, 0); @@ -862,8 +901,12 @@ static int k3_dma_probe(struct platform_device *op) return -ENOMEM; for (i = 0; i < d->dma_channels; i++) { - struct k3_dma_phy *p = &d->phy[i]; + struct k3_dma_phy *p; + + if (!(d->dma_channel_mask & BIT(i))) + continue; + p = &d->phy[i]; p->idx = i; p->base = d->base + i * 0x40; } diff --git a/drivers/dma/mcf-edma.c b/drivers/dma/mcf-edma.c index 5de1b07eddff4..7de54b2fafdb5 100644 --- a/drivers/dma/mcf-edma.c +++ b/drivers/dma/mcf-edma.c @@ -214,6 +214,7 @@ static int mcf_edma_probe(struct platform_device *pdev) mcf_chan->edma = mcf_edma; mcf_chan->slave_id = i; mcf_chan->idle = true; + mcf_chan->dma_dir = DMA_NONE; mcf_chan->vchan.desc_free = fsl_edma_free_desc; vchan_init(&mcf_chan->vchan, &mcf_edma->dma_dev); iowrite32(0x0, ®s->tcd[i].csr); diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index 7f595355fb793..65af2e7fcb2c4 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -1059,6 +1059,7 @@ mv_xor_channel_add(struct mv_xor_device *xordev, mv_chan->op_in_desc = XOR_MODE_IN_DESC; dma_dev = &mv_chan->dmadev; + dma_dev->dev = &pdev->dev; mv_chan->xordev = xordev; /* @@ -1091,7 +1092,6 @@ mv_xor_channel_add(struct mv_xor_device *xordev, dma_dev->device_free_chan_resources = mv_xor_free_chan_resources; dma_dev->device_tx_status = mv_xor_status; dma_dev->device_issue_pending = mv_xor_issue_pending; - dma_dev->dev = &pdev->dev; /* set prep routines based on capability */ if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) @@ -1153,7 +1153,10 @@ mv_xor_channel_add(struct mv_xor_device *xordev, dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); - dma_async_device_register(dma_dev); + ret = dma_async_device_register(dma_dev); + if (ret) + goto err_free_irq; + return mv_chan; err_free_irq: diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index cff1b143fff5d..eec79fdf27a5b 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -2267,7 +2267,6 @@ static int pl330_terminate_all(struct dma_chan *chan) struct dma_pl330_desc *desc; unsigned long flags; struct pl330_dmac *pl330 = pch->dmac; - LIST_HEAD(list); bool power_down = false; pm_runtime_get_sync(pl330->ddma.dev); diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 1617715aa6e07..cb860cb53c274 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -636,8 +636,8 @@ static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan, num_alloc += DIV_ROUND_UP(sg_dma_len(sg), BAM_FIFO_SIZE); /* allocate enough room to accomodate the number of entries */ - async_desc = kzalloc(sizeof(*async_desc) + - (num_alloc * sizeof(struct bam_desc_hw)), GFP_NOWAIT); + async_desc = kzalloc(struct_size(async_desc, desc, num_alloc), + GFP_NOWAIT); if (!async_desc) goto err_out; diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index 43d4b00b81388..411f91fde7345 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -138,24 +138,25 @@ static void hidma_process_completed(struct hidma_chan *mchan) desc = &mdesc->desc; last_cookie = desc->cookie; + llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); + spin_lock_irqsave(&mchan->lock, irqflags); + if (llstat == DMA_COMPLETE) { + mchan->last_success = last_cookie; + result.result = DMA_TRANS_NOERROR; + } else { + result.result = DMA_TRANS_ABORTED; + } + dma_cookie_complete(desc); spin_unlock_irqrestore(&mchan->lock, irqflags); - llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); dmaengine_desc_get_callback(desc, &cb); dma_run_dependencies(desc); spin_lock_irqsave(&mchan->lock, irqflags); list_move(&mdesc->node, &mchan->free); - - if (llstat == DMA_COMPLETE) { - mchan->last_success = last_cookie; - result.result = DMA_TRANS_NOERROR; - } else - result.result = DMA_TRANS_ABORTED; - spin_unlock_irqrestore(&mchan->lock, irqflags); dmaengine_desc_callback_invoke(&cb, &result); @@ -415,6 +416,7 @@ hidma_prep_dma_memcpy(struct dma_chan *dmach, dma_addr_t dest, dma_addr_t src, if (!mdesc) return NULL; + mdesc->desc.flags = flags; hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, src, dest, len, flags, HIDMA_TRE_MEMCPY); @@ -447,6 +449,7 @@ hidma_prep_dma_memset(struct dma_chan *dmach, dma_addr_t dest, int value, if (!mdesc) return NULL; + mdesc->desc.flags = flags; hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, value, dest, len, flags, HIDMA_TRE_MEMSET); diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c index d64edeb6771a9..681de12f4c679 100644 --- a/drivers/dma/qcom/hidma_mgmt.c +++ b/drivers/dma/qcom/hidma_mgmt.c @@ -423,9 +423,8 @@ static int __init hidma_mgmt_init(void) hidma_mgmt_of_populate_channels(child); } #endif - platform_driver_register(&hidma_mgmt_driver); + return platform_driver_register(&hidma_mgmt_driver); - return 0; } module_init(hidma_mgmt_init); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/sa11x0-dma.c b/drivers/dma/sa11x0-dma.c index 784d5f1a473bf..3fae23768b478 100644 --- a/drivers/dma/sa11x0-dma.c +++ b/drivers/dma/sa11x0-dma.c @@ -705,7 +705,6 @@ static int sa11x0_dma_device_pause(struct dma_chan *chan) struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan); struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device); struct sa11x0_dma_phy *p; - LIST_HEAD(head); unsigned long flags; dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc); @@ -732,7 +731,6 @@ static int sa11x0_dma_device_resume(struct dma_chan *chan) struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan); struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device); struct sa11x0_dma_phy *p; - LIST_HEAD(head); unsigned long flags; dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc); diff --git a/drivers/dma/sh/usb-dmac.c b/drivers/dma/sh/usb-dmac.c index 7f7184c3cf95f..59403f6d008a3 100644 --- a/drivers/dma/sh/usb-dmac.c +++ b/drivers/dma/sh/usb-dmac.c @@ -694,6 +694,8 @@ static int usb_dmac_runtime_resume(struct device *dev) #endif /* CONFIG_PM */ static const struct dev_pm_ops usb_dmac_pm = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) SET_RUNTIME_PM_OPS(usb_dmac_runtime_suspend, usb_dmac_runtime_resume, NULL) }; diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c index e2f016700fcca..48431e2da9876 100644 --- a/drivers/dma/sprd-dma.c +++ b/drivers/dma/sprd-dma.c @@ -580,15 +580,7 @@ static irqreturn_t dma_irq_handle(int irq, void *dev_id) static int sprd_dma_alloc_chan_resources(struct dma_chan *chan) { - struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); - int ret; - - ret = pm_runtime_get_sync(chan->device->dev); - if (ret < 0) - return ret; - - schan->dev_id = SPRD_DMA_SOFTWARE_UID; - return 0; + return pm_runtime_get_sync(chan->device->dev); } static void sprd_dma_free_chan_resources(struct dma_chan *chan) @@ -1021,13 +1013,10 @@ static void sprd_dma_free_desc(struct virt_dma_desc *vd) static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param) { struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); - struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); - u32 req = *(u32 *)param; + u32 slave_id = *(u32 *)param; - if (req < sdev->total_chns) - return req == schan->chn_num + 1; - else - return false; + schan->dev_id = slave_id; + return true; } static int sprd_dma_probe(struct platform_device *pdev) diff --git a/drivers/dma/st_fdma.c b/drivers/dma/st_fdma.c index 07c20aa2e955a..bc7a1de3f29b7 100644 --- a/drivers/dma/st_fdma.c +++ b/drivers/dma/st_fdma.c @@ -243,8 +243,7 @@ static struct st_fdma_desc *st_fdma_alloc_desc(struct st_fdma_chan *fchan, struct st_fdma_desc *fdesc; int i; - fdesc = kzalloc(sizeof(*fdesc) + - sizeof(struct st_fdma_sw_node) * sg_len, GFP_NOWAIT); + fdesc = kzalloc(struct_size(fdesc, node, sg_len), GFP_NOWAIT); if (!fdesc) return NULL; @@ -294,8 +293,6 @@ static void st_fdma_free_chan_res(struct dma_chan *chan) struct rproc *rproc = fchan->fdev->slim_rproc->rproc; unsigned long flags; - LIST_HEAD(head); - dev_dbg(fchan->fdev->dev, "%s: freeing chan:%d\n", __func__, fchan->vchan.chan.chan_id); @@ -626,7 +623,6 @@ static void st_fdma_issue_pending(struct dma_chan *chan) static int st_fdma_pause(struct dma_chan *chan) { unsigned long flags; - LIST_HEAD(head); struct st_fdma_chan *fchan = to_st_fdma_chan(chan); int ch_id = fchan->vchan.chan.chan_id; unsigned long cmd = FDMA_CMD_PAUSE(ch_id); diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c index 4903a408fc146..ba239b529fa9e 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32-dma.c @@ -23,6 +23,7 @@ #include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> +#include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/sched.h> #include <linux/slab.h> @@ -641,12 +642,13 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid) { struct stm32_dma_chan *chan = devid; struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); - u32 status, scr; + u32 status, scr, sfcr; spin_lock(&chan->vchan.lock); status = stm32_dma_irq_status(chan); scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); + sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id)); if (status & STM32_DMA_TCI) { stm32_dma_irq_clear(chan, STM32_DMA_TCI); @@ -661,10 +663,12 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid) if (status & STM32_DMA_FEI) { stm32_dma_irq_clear(chan, STM32_DMA_FEI); status &= ~STM32_DMA_FEI; - if (!(scr & STM32_DMA_SCR_EN)) - dev_err(chan2dev(chan), "FIFO Error\n"); - else - dev_dbg(chan2dev(chan), "FIFO over/underrun\n"); + if (sfcr & STM32_DMA_SFCR_FEIE) { + if (!(scr & STM32_DMA_SCR_EN)) + dev_err(chan2dev(chan), "FIFO Error\n"); + else + dev_dbg(chan2dev(chan), "FIFO over/underrun\n"); + } } if (status) { stm32_dma_irq_clear(chan, status); @@ -1112,15 +1116,14 @@ static int stm32_dma_alloc_chan_resources(struct dma_chan *c) int ret; chan->config_init = false; - ret = clk_prepare_enable(dmadev->clk); - if (ret < 0) { - dev_err(chan2dev(chan), "clk_prepare_enable failed: %d\n", ret); + + ret = pm_runtime_get_sync(dmadev->ddev.dev); + if (ret < 0) return ret; - } ret = stm32_dma_disable_chan(chan); if (ret < 0) - clk_disable_unprepare(dmadev->clk); + pm_runtime_put(dmadev->ddev.dev); return ret; } @@ -1140,7 +1143,7 @@ static void stm32_dma_free_chan_resources(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); } - clk_disable_unprepare(dmadev->clk); + pm_runtime_put(dmadev->ddev.dev); vchan_free_chan_resources(to_virt_chan(c)); } @@ -1240,6 +1243,12 @@ static int stm32_dma_probe(struct platform_device *pdev) return PTR_ERR(dmadev->clk); } + ret = clk_prepare_enable(dmadev->clk); + if (ret < 0) { + dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret); + return ret; + } + dmadev->mem2mem = of_property_read_bool(pdev->dev.of_node, "st,mem2mem"); @@ -1289,7 +1298,7 @@ static int stm32_dma_probe(struct platform_device *pdev) ret = dma_async_device_register(dd); if (ret) - return ret; + goto clk_free; for (i = 0; i < STM32_DMA_MAX_CHANNELS; i++) { chan = &dmadev->chan[i]; @@ -1321,20 +1330,58 @@ static int stm32_dma_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dmadev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_put(&pdev->dev); + dev_info(&pdev->dev, "STM32 DMA driver registered\n"); return 0; err_unregister: dma_async_device_unregister(dd); +clk_free: + clk_disable_unprepare(dmadev->clk); return ret; } +#ifdef CONFIG_PM +static int stm32_dma_runtime_suspend(struct device *dev) +{ + struct stm32_dma_device *dmadev = dev_get_drvdata(dev); + + clk_disable_unprepare(dmadev->clk); + + return 0; +} + +static int stm32_dma_runtime_resume(struct device *dev) +{ + struct stm32_dma_device *dmadev = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(dmadev->clk); + if (ret) { + dev_err(dev, "failed to prepare_enable clock\n"); + return ret; + } + + return 0; +} +#endif + +static const struct dev_pm_ops stm32_dma_pm_ops = { + SET_RUNTIME_PM_OPS(stm32_dma_runtime_suspend, + stm32_dma_runtime_resume, NULL) +}; + static struct platform_driver stm32_dma_driver = { .driver = { .name = "stm32-dma", .of_match_table = stm32_dma_of_match, + .pm = &stm32_dma_pm_ops, }, }; diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32-dmamux.c index b922db90939a5..a67119199c455 100644 --- a/drivers/dma/stm32-dmamux.c +++ b/drivers/dma/stm32-dmamux.c @@ -28,6 +28,7 @@ #include <linux/module.h> #include <linux/of_device.h> #include <linux/of_dma.h> +#include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/slab.h> #include <linux/spinlock.h> @@ -79,8 +80,7 @@ static void stm32_dmamux_free(struct device *dev, void *route_data) stm32_dmamux_write(dmamux->iomem, STM32_DMAMUX_CCR(mux->chan_id), 0); clear_bit(mux->chan_id, dmamux->dma_inuse); - if (!IS_ERR(dmamux->clk)) - clk_disable(dmamux->clk); + pm_runtime_put_sync(dev); spin_unlock_irqrestore(&dmamux->lock, flags); @@ -146,13 +146,10 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec, /* Set dma request */ spin_lock_irqsave(&dmamux->lock, flags); - if (!IS_ERR(dmamux->clk)) { - ret = clk_enable(dmamux->clk); - if (ret < 0) { - spin_unlock_irqrestore(&dmamux->lock, flags); - dev_err(&pdev->dev, "clk_prep_enable issue: %d\n", ret); - goto error; - } + ret = pm_runtime_get_sync(&pdev->dev); + if (ret < 0) { + spin_unlock_irqrestore(&dmamux->lock, flags); + goto error; } spin_unlock_irqrestore(&dmamux->lock, flags); @@ -254,6 +251,7 @@ static int stm32_dmamux_probe(struct platform_device *pdev) dev_warn(&pdev->dev, "DMAMUX defaulting on %u requests\n", stm32_dmamux->dmamux_requests); } + pm_runtime_get_noresume(&pdev->dev); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); iomem = devm_ioremap_resource(&pdev->dev, res); @@ -282,6 +280,8 @@ static int stm32_dmamux_probe(struct platform_device *pdev) stm32_dmamux->dmarouter.route_free = stm32_dmamux_free; platform_set_drvdata(pdev, stm32_dmamux); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); if (!IS_ERR(stm32_dmamux->clk)) { ret = clk_prepare_enable(stm32_dmamux->clk); @@ -291,17 +291,52 @@ static int stm32_dmamux_probe(struct platform_device *pdev) } } + pm_runtime_get_noresume(&pdev->dev); + /* Reset the dmamux */ for (i = 0; i < stm32_dmamux->dma_requests; i++) stm32_dmamux_write(stm32_dmamux->iomem, STM32_DMAMUX_CCR(i), 0); - if (!IS_ERR(stm32_dmamux->clk)) - clk_disable(stm32_dmamux->clk); + pm_runtime_put(&pdev->dev); return of_dma_router_register(node, stm32_dmamux_route_allocate, &stm32_dmamux->dmarouter); } +#ifdef CONFIG_PM +static int stm32_dmamux_runtime_suspend(struct device *dev) +{ + struct platform_device *pdev = + container_of(dev, struct platform_device, dev); + struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev); + + clk_disable_unprepare(stm32_dmamux->clk); + + return 0; +} + +static int stm32_dmamux_runtime_resume(struct device *dev) +{ + struct platform_device *pdev = + container_of(dev, struct platform_device, dev); + struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev); + int ret; + + ret = clk_prepare_enable(stm32_dmamux->clk); + if (ret) { + dev_err(&pdev->dev, "failed to prepare_enable clock\n"); + return ret; + } + + return 0; +} +#endif + +static const struct dev_pm_ops stm32_dmamux_pm_ops = { + SET_RUNTIME_PM_OPS(stm32_dmamux_runtime_suspend, + stm32_dmamux_runtime_resume, NULL) +}; + static const struct of_device_id stm32_dmamux_match[] = { { .compatible = "st,stm32h7-dmamux" }, {}, @@ -312,6 +347,7 @@ static struct platform_driver stm32_dmamux_driver = { .driver = { .name = "stm32-dmamux", .of_match_table = stm32_dmamux_match, + .pm = &stm32_dmamux_pm_ops, }, }; diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c index 390e4cae0e1a5..4e0eede599a8d 100644 --- a/drivers/dma/stm32-mdma.c +++ b/drivers/dma/stm32-mdma.c @@ -37,6 +37,7 @@ #include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> +#include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/slab.h> @@ -1456,15 +1457,13 @@ static int stm32_mdma_alloc_chan_resources(struct dma_chan *c) return -ENOMEM; } - ret = clk_prepare_enable(dmadev->clk); - if (ret < 0) { - dev_err(chan2dev(chan), "clk_prepare_enable failed: %d\n", ret); + ret = pm_runtime_get_sync(dmadev->ddev.dev); + if (ret < 0) return ret; - } ret = stm32_mdma_disable_chan(chan); if (ret < 0) - clk_disable_unprepare(dmadev->clk); + pm_runtime_put(dmadev->ddev.dev); return ret; } @@ -1484,7 +1483,7 @@ static void stm32_mdma_free_chan_resources(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); } - clk_disable_unprepare(dmadev->clk); + pm_runtime_put(dmadev->ddev.dev); vchan_free_chan_resources(to_virt_chan(c)); dmam_pool_destroy(chan->desc_pool); chan->desc_pool = NULL; @@ -1579,9 +1578,11 @@ static int stm32_mdma_probe(struct platform_device *pdev) dmadev->nr_channels = nr_channels; dmadev->nr_requests = nr_requests; - device_property_read_u32_array(&pdev->dev, "st,ahb-addr-masks", + ret = device_property_read_u32_array(&pdev->dev, "st,ahb-addr-masks", dmadev->ahb_addr_masks, count); + if (ret) + return ret; dmadev->nr_ahb_addr_masks = count; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -1597,6 +1598,12 @@ static int stm32_mdma_probe(struct platform_device *pdev) return ret; } + ret = clk_prepare_enable(dmadev->clk); + if (ret < 0) { + dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret); + return ret; + } + dmadev->rst = devm_reset_control_get(&pdev->dev, NULL); if (!IS_ERR(dmadev->rst)) { reset_control_assert(dmadev->rst); @@ -1668,6 +1675,10 @@ static int stm32_mdma_probe(struct platform_device *pdev) } platform_set_drvdata(pdev, dmadev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_put(&pdev->dev); dev_info(&pdev->dev, "STM32 MDMA driver registered\n"); @@ -1677,11 +1688,42 @@ err_unregister: return ret; } +#ifdef CONFIG_PM +static int stm32_mdma_runtime_suspend(struct device *dev) +{ + struct stm32_mdma_device *dmadev = dev_get_drvdata(dev); + + clk_disable_unprepare(dmadev->clk); + + return 0; +} + +static int stm32_mdma_runtime_resume(struct device *dev) +{ + struct stm32_mdma_device *dmadev = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(dmadev->clk); + if (ret) { + dev_err(dev, "failed to prepare_enable clock\n"); + return ret; + } + + return 0; +} +#endif + +static const struct dev_pm_ops stm32_mdma_pm_ops = { + SET_RUNTIME_PM_OPS(stm32_mdma_runtime_suspend, + stm32_mdma_runtime_resume, NULL) +}; + static struct platform_driver stm32_mdma_driver = { .probe = stm32_mdma_probe, .driver = { .name = "stm32-mdma", .of_match_table = stm32_mdma_of_match, + .pm = &stm32_mdma_pm_ops, }, }; diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index 9a558e30c461c..cf462b1abc0bb 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -38,6 +38,9 @@ #include "dmaengine.h" +#define CREATE_TRACE_POINTS +#include <trace/events/tegra_apb_dma.h> + #define TEGRA_APBDMA_GENERAL 0x0 #define TEGRA_APBDMA_GENERAL_ENABLE BIT(31) @@ -146,7 +149,7 @@ struct tegra_dma_channel_regs { }; /* - * tegra_dma_sg_req: Dma request details to configure hardware. This + * tegra_dma_sg_req: DMA request details to configure hardware. This * contains the details for one transfer to configure DMA hw. * The client's request for data transfer can be broken into multiple * sub-transfer as per requester details and hw support. @@ -155,7 +158,7 @@ struct tegra_dma_channel_regs { */ struct tegra_dma_sg_req { struct tegra_dma_channel_regs ch_regs; - int req_len; + unsigned int req_len; bool configured; bool last_sg; struct list_head node; @@ -169,8 +172,8 @@ struct tegra_dma_sg_req { */ struct tegra_dma_desc { struct dma_async_tx_descriptor txd; - int bytes_requested; - int bytes_transferred; + unsigned int bytes_requested; + unsigned int bytes_transferred; enum dma_status dma_status; struct list_head node; struct list_head tx_list; @@ -186,7 +189,7 @@ typedef void (*dma_isr_handler)(struct tegra_dma_channel *tdc, /* tegra_dma_channel: Channel specific information */ struct tegra_dma_channel { struct dma_chan dma_chan; - char name[30]; + char name[12]; bool config_init; int id; int irq; @@ -574,7 +577,7 @@ static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, struct tegra_dma_sg_req *hsgreq = NULL; if (list_empty(&tdc->pending_sg_req)) { - dev_err(tdc2dev(tdc), "Dma is running without req\n"); + dev_err(tdc2dev(tdc), "DMA is running without req\n"); tegra_dma_stop(tdc); return false; } @@ -587,7 +590,7 @@ static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); if (!hsgreq->configured) { tegra_dma_stop(tdc); - dev_err(tdc2dev(tdc), "Error in dma transfer, aborting dma\n"); + dev_err(tdc2dev(tdc), "Error in DMA transfer, aborting DMA\n"); tegra_dma_abort_all(tdc); return false; } @@ -636,7 +639,10 @@ static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); dma_desc = sgreq->dma_desc; - dma_desc->bytes_transferred += sgreq->req_len; + /* if we dma for long enough the transfer count will wrap */ + dma_desc->bytes_transferred = + (dma_desc->bytes_transferred + sgreq->req_len) % + dma_desc->bytes_requested; /* Callback need to be call */ if (!dma_desc->cb_count) @@ -669,6 +675,8 @@ static void tegra_dma_tasklet(unsigned long data) dmaengine_desc_get_callback(&dma_desc->txd, &cb); cb_count = dma_desc->cb_count; dma_desc->cb_count = 0; + trace_tegra_dma_complete_cb(&tdc->dma_chan, cb_count, + cb.callback); spin_unlock_irqrestore(&tdc->lock, flags); while (cb_count--) dmaengine_desc_callback_invoke(&cb, NULL); @@ -685,6 +693,7 @@ static irqreturn_t tegra_dma_isr(int irq, void *dev_id) spin_lock_irqsave(&tdc->lock, flags); + trace_tegra_dma_isr(&tdc->dma_chan, irq); status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status); @@ -843,6 +852,7 @@ found: dma_set_residue(txstate, residual); } + trace_tegra_dma_tx_status(&tdc->dma_chan, cookie, txstate); spin_unlock_irqrestore(&tdc->lock, flags); return ret; } @@ -919,7 +929,7 @@ static int get_transfer_param(struct tegra_dma_channel *tdc, return 0; default: - dev_err(tdc2dev(tdc), "Dma direction is not supported\n"); + dev_err(tdc2dev(tdc), "DMA direction is not supported\n"); return -EINVAL; } return -EINVAL; @@ -952,7 +962,7 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( enum dma_slave_buswidth slave_bw; if (!tdc->config_init) { - dev_err(tdc2dev(tdc), "dma channel is not configured\n"); + dev_err(tdc2dev(tdc), "DMA channel is not configured\n"); return NULL; } if (sg_len < 1) { @@ -985,7 +995,7 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( dma_desc = tegra_dma_desc_get(tdc); if (!dma_desc) { - dev_err(tdc2dev(tdc), "Dma descriptors not available\n"); + dev_err(tdc2dev(tdc), "DMA descriptors not available\n"); return NULL; } INIT_LIST_HEAD(&dma_desc->tx_list); @@ -1005,14 +1015,14 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( if ((len & 3) || (mem & 3) || (len > tdc->tdma->chip_data->max_dma_count)) { dev_err(tdc2dev(tdc), - "Dma length/memory address is not supported\n"); + "DMA length/memory address is not supported\n"); tegra_dma_desc_put(tdc, dma_desc); return NULL; } sg_req = tegra_dma_sg_req_get(tdc); if (!sg_req) { - dev_err(tdc2dev(tdc), "Dma sg-req not available\n"); + dev_err(tdc2dev(tdc), "DMA sg-req not available\n"); tegra_dma_desc_put(tdc, dma_desc); return NULL; } @@ -1087,7 +1097,7 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( * terminating the DMA. */ if (tdc->busy) { - dev_err(tdc2dev(tdc), "Request not allowed when dma running\n"); + dev_err(tdc2dev(tdc), "Request not allowed when DMA running\n"); return NULL; } @@ -1144,7 +1154,7 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( while (remain_len) { sg_req = tegra_dma_sg_req_get(tdc); if (!sg_req) { - dev_err(tdc2dev(tdc), "Dma sg-req not available\n"); + dev_err(tdc2dev(tdc), "DMA sg-req not available\n"); tegra_dma_desc_put(tdc, dma_desc); return NULL; } @@ -1319,8 +1329,9 @@ static int tegra_dma_probe(struct platform_device *pdev) return -ENODEV; } - tdma = devm_kzalloc(&pdev->dev, sizeof(*tdma) + cdata->nr_channels * - sizeof(struct tegra_dma_channel), GFP_KERNEL); + tdma = devm_kzalloc(&pdev->dev, + struct_size(tdma, channels, cdata->nr_channels), + GFP_KERNEL); if (!tdma) return -ENOMEM; diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index b26256f23d67f..5ec0dd97b3971 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -678,8 +678,9 @@ static int tegra_adma_probe(struct platform_device *pdev) return -ENODEV; } - tdma = devm_kzalloc(&pdev->dev, sizeof(*tdma) + cdata->nr_channels * - sizeof(struct tegra_adma_chan), GFP_KERNEL); + tdma = devm_kzalloc(&pdev->dev, + struct_size(tdma, channels, cdata->nr_channels), + GFP_KERNEL); if (!tdma) return -ENOMEM; diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c index fc0f9c8766a87..afbb1c95b721b 100644 --- a/drivers/dma/timb_dma.c +++ b/drivers/dma/timb_dma.c @@ -643,8 +643,8 @@ static int td_probe(struct platform_device *pdev) DRIVER_NAME)) return -EBUSY; - td = kzalloc(sizeof(struct timb_dma) + - sizeof(struct timb_dma_chan) * pdata->nr_channels, GFP_KERNEL); + td = kzalloc(struct_size(td, channels, pdata->nr_channels), + GFP_KERNEL); if (!td) { err = -ENOMEM; goto err_release_region; diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index cb20b411493e4..c43c1a154604b 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -86,6 +86,7 @@ #define XILINX_DMA_DMASR_DMA_DEC_ERR BIT(6) #define XILINX_DMA_DMASR_DMA_SLAVE_ERR BIT(5) #define XILINX_DMA_DMASR_DMA_INT_ERR BIT(4) +#define XILINX_DMA_DMASR_SG_MASK BIT(3) #define XILINX_DMA_DMASR_IDLE BIT(1) #define XILINX_DMA_DMASR_HALTED BIT(0) #define XILINX_DMA_DMASR_DELAY_MASK GENMASK(31, 24) @@ -161,7 +162,9 @@ #define XILINX_DMA_REG_BTT 0x28 /* AXI DMA Specific Masks/Bit fields */ -#define XILINX_DMA_MAX_TRANS_LEN GENMASK(22, 0) +#define XILINX_DMA_MAX_TRANS_LEN_MIN 8 +#define XILINX_DMA_MAX_TRANS_LEN_MAX 23 +#define XILINX_DMA_V2_MAX_TRANS_LEN_MAX 26 #define XILINX_DMA_CR_COALESCE_MAX GENMASK(23, 16) #define XILINX_DMA_CR_CYCLIC_BD_EN_MASK BIT(4) #define XILINX_DMA_CR_COALESCE_SHIFT 16 @@ -412,7 +415,6 @@ struct xilinx_dma_config { * @dev: Device Structure * @common: DMA device structure * @chan: Driver specific DMA channel - * @has_sg: Specifies whether Scatter-Gather is present or not * @mcdma: Specifies whether Multi-Channel is present or not * @flush_on_fsync: Flush on frame sync * @ext_addr: Indicates 64 bit addressing is supported by dma device @@ -425,13 +427,13 @@ struct xilinx_dma_config { * @rxs_clk: DMA s2mm stream clock * @nr_channels: Number of channels DMA device supports * @chan_id: DMA channel identifier + * @max_buffer_len: Max buffer length */ struct xilinx_dma_device { void __iomem *regs; struct device *dev; struct dma_device common; struct xilinx_dma_chan *chan[XILINX_DMA_MAX_CHANS_PER_DEVICE]; - bool has_sg; bool mcdma; u32 flush_on_fsync; bool ext_addr; @@ -444,6 +446,7 @@ struct xilinx_dma_device { struct clk *rxs_clk; u32 nr_channels; u32 chan_id; + u32 max_buffer_len; }; /* Macros */ @@ -960,6 +963,34 @@ static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan) } /** + * xilinx_dma_calc_copysize - Calculate the amount of data to copy + * @chan: Driver specific DMA channel + * @size: Total data that needs to be copied + * @done: Amount of data that has been already copied + * + * Return: Amount of data that has to be copied + */ +static int xilinx_dma_calc_copysize(struct xilinx_dma_chan *chan, + int size, int done) +{ + size_t copy; + + copy = min_t(size_t, size - done, + chan->xdev->max_buffer_len); + + if ((copy + done < size) && + chan->xdev->common.copy_align) { + /* + * If this is not the last descriptor, make sure + * the next one will be properly aligned + */ + copy = rounddown(copy, + (1 << chan->xdev->common.copy_align)); + } + return copy; +} + +/** * xilinx_dma_tx_status - Get DMA transaction status * @dchan: DMA channel * @cookie: Transaction identifier @@ -992,7 +1023,7 @@ static enum dma_status xilinx_dma_tx_status(struct dma_chan *dchan, list_for_each_entry(segment, &desc->segments, node) { hw = &segment->hw; residue += (hw->control - hw->status) & - XILINX_DMA_MAX_TRANS_LEN; + chan->xdev->max_buffer_len; } } spin_unlock_irqrestore(&chan->lock, flags); @@ -1070,7 +1101,8 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) struct xilinx_vdma_config *config = &chan->config; struct xilinx_dma_tx_descriptor *desc, *tail_desc; u32 reg, j; - struct xilinx_vdma_tx_segment *tail_segment; + struct xilinx_vdma_tx_segment *segment, *last = NULL; + int i = 0; /* This function was invoked with lock held */ if (chan->err) @@ -1087,17 +1119,6 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) tail_desc = list_last_entry(&chan->pending_list, struct xilinx_dma_tx_descriptor, node); - tail_segment = list_last_entry(&tail_desc->segments, - struct xilinx_vdma_tx_segment, node); - - /* - * If hardware is idle, then all descriptors on the running lists are - * done, start new transfers - */ - if (chan->has_sg) - dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC, - desc->async_tx.phys); - /* Configure the hardware using info in the config structure */ if (chan->has_vflip) { reg = dma_read(chan, XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP); @@ -1114,15 +1135,11 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) else reg &= ~XILINX_DMA_DMACR_FRAMECNT_EN; - /* - * With SG, start with circular mode, so that BDs can be fetched. - * In direct register mode, if not parking, enable circular mode - */ - if (chan->has_sg || !config->park) - reg |= XILINX_DMA_DMACR_CIRC_EN; - + /* If not parking, enable circular mode */ if (config->park) reg &= ~XILINX_DMA_DMACR_CIRC_EN; + else + reg |= XILINX_DMA_DMACR_CIRC_EN; dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); @@ -1144,48 +1161,38 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) return; /* Start the transfer */ - if (chan->has_sg) { - dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC, - tail_segment->phys); - list_splice_tail_init(&chan->pending_list, &chan->active_list); - chan->desc_pendingcount = 0; - } else { - struct xilinx_vdma_tx_segment *segment, *last = NULL; - int i = 0; - - if (chan->desc_submitcount < chan->num_frms) - i = chan->desc_submitcount; - - list_for_each_entry(segment, &desc->segments, node) { - if (chan->ext_addr) - vdma_desc_write_64(chan, - XILINX_VDMA_REG_START_ADDRESS_64(i++), - segment->hw.buf_addr, - segment->hw.buf_addr_msb); - else - vdma_desc_write(chan, + if (chan->desc_submitcount < chan->num_frms) + i = chan->desc_submitcount; + + list_for_each_entry(segment, &desc->segments, node) { + if (chan->ext_addr) + vdma_desc_write_64(chan, + XILINX_VDMA_REG_START_ADDRESS_64(i++), + segment->hw.buf_addr, + segment->hw.buf_addr_msb); + else + vdma_desc_write(chan, XILINX_VDMA_REG_START_ADDRESS(i++), segment->hw.buf_addr); - last = segment; - } + last = segment; + } - if (!last) - return; + if (!last) + return; - /* HW expects these parameters to be same for one transaction */ - vdma_desc_write(chan, XILINX_DMA_REG_HSIZE, last->hw.hsize); - vdma_desc_write(chan, XILINX_DMA_REG_FRMDLY_STRIDE, - last->hw.stride); - vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize); + /* HW expects these parameters to be same for one transaction */ + vdma_desc_write(chan, XILINX_DMA_REG_HSIZE, last->hw.hsize); + vdma_desc_write(chan, XILINX_DMA_REG_FRMDLY_STRIDE, + last->hw.stride); + vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize); - chan->desc_submitcount++; - chan->desc_pendingcount--; - list_del(&desc->node); - list_add_tail(&desc->node, &chan->active_list); - if (chan->desc_submitcount == chan->num_frms) - chan->desc_submitcount = 0; - } + chan->desc_submitcount++; + chan->desc_pendingcount--; + list_del(&desc->node); + list_add_tail(&desc->node, &chan->active_list); + if (chan->desc_submitcount == chan->num_frms) + chan->desc_submitcount = 0; chan->idle = false; } @@ -1254,7 +1261,7 @@ static void xilinx_cdma_start_transfer(struct xilinx_dma_chan *chan) /* Start the transfer */ dma_ctrl_write(chan, XILINX_DMA_REG_BTT, - hw->control & XILINX_DMA_MAX_TRANS_LEN); + hw->control & chan->xdev->max_buffer_len); } list_splice_tail_init(&chan->pending_list, &chan->active_list); @@ -1357,7 +1364,7 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) /* Start the transfer */ dma_ctrl_write(chan, XILINX_DMA_REG_BTT, - hw->control & XILINX_DMA_MAX_TRANS_LEN); + hw->control & chan->xdev->max_buffer_len); } list_splice_tail_init(&chan->pending_list, &chan->active_list); @@ -1718,7 +1725,7 @@ xilinx_cdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst, struct xilinx_cdma_tx_segment *segment; struct xilinx_cdma_desc_hw *hw; - if (!len || len > XILINX_DMA_MAX_TRANS_LEN) + if (!len || len > chan->xdev->max_buffer_len) return NULL; desc = xilinx_dma_alloc_tx_descriptor(chan); @@ -1808,8 +1815,8 @@ static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg( * Calculate the maximum number of bytes to transfer, * making sure it is less than the hw limit */ - copy = min_t(size_t, sg_dma_len(sg) - sg_used, - XILINX_DMA_MAX_TRANS_LEN); + copy = xilinx_dma_calc_copysize(chan, sg_dma_len(sg), + sg_used); hw = &segment->hw; /* Fill in the descriptor */ @@ -1913,8 +1920,8 @@ static struct dma_async_tx_descriptor *xilinx_dma_prep_dma_cyclic( * Calculate the maximum number of bytes to transfer, * making sure it is less than the hw limit */ - copy = min_t(size_t, period_len - sg_used, - XILINX_DMA_MAX_TRANS_LEN); + copy = xilinx_dma_calc_copysize(chan, period_len, + sg_used); hw = &segment->hw; xilinx_axidma_buf(chan, hw, buf_addr, sg_used, period_len * i); @@ -2389,7 +2396,6 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, chan->dev = xdev->dev; chan->xdev = xdev; - chan->has_sg = xdev->has_sg; chan->desc_pendingcount = 0x0; chan->ext_addr = xdev->ext_addr; /* This variable ensures that descriptors are not @@ -2489,6 +2495,15 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, chan->stop_transfer = xilinx_dma_stop_transfer; } + /* check if SG is enabled (only for AXIDMA and CDMA) */ + if (xdev->dma_config->dmatype != XDMA_TYPE_VDMA) { + if (dma_ctrl_read(chan, XILINX_DMA_REG_DMASR) & + XILINX_DMA_DMASR_SG_MASK) + chan->has_sg = true; + dev_dbg(chan->dev, "ch %d: SG %s\n", chan->id, + chan->has_sg ? "enabled" : "disabled"); + } + /* Initialize the tasklet */ tasklet_init(&chan->tasklet, xilinx_dma_do_tasklet, (unsigned long)chan); @@ -2596,7 +2611,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) struct xilinx_dma_device *xdev; struct device_node *child, *np = pdev->dev.of_node; struct resource *io; - u32 num_frames, addr_width; + u32 num_frames, addr_width, len_width; int i, err; /* Allocate and initialize the DMA engine structure */ @@ -2627,9 +2642,24 @@ static int xilinx_dma_probe(struct platform_device *pdev) return PTR_ERR(xdev->regs); /* Retrieve the DMA engine properties from the device tree */ - xdev->has_sg = of_property_read_bool(node, "xlnx,include-sg"); - if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) + xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0); + + if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { xdev->mcdma = of_property_read_bool(node, "xlnx,mcdma"); + if (!of_property_read_u32(node, "xlnx,sg-length-width", + &len_width)) { + if (len_width < XILINX_DMA_MAX_TRANS_LEN_MIN || + len_width > XILINX_DMA_V2_MAX_TRANS_LEN_MAX) { + dev_warn(xdev->dev, + "invalid xlnx,sg-length-width property value. Using default width\n"); + } else { + if (len_width > XILINX_DMA_MAX_TRANS_LEN_MAX) + dev_warn(xdev->dev, "Please ensure that IP supports buffer length > 23 bits\n"); + xdev->max_buffer_len = + GENMASK(len_width - 1, 0); + } + } + } if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { err = of_property_read_u32(node, "xlnx,num-fstores", diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index ead851413c0aa..bfa9062ce6b9f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -947,10 +947,6 @@ int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, if (r) return r; - r = amdgpu_vm_clear_bo(adev, vm, pt, cursor.level, ats); - if (r) - goto error_free_pt; - if (vm->use_cpu_for_update) { r = amdgpu_bo_kmap(pt, NULL); if (r) @@ -963,6 +959,10 @@ int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, pt->parent = amdgpu_bo_ref(cursor.parent->base.bo); amdgpu_vm_bo_base_init(&entry->base, vm, pt); + + r = amdgpu_vm_clear_bo(adev, vm, pt, cursor.level, ats); + if (r) + goto error_free_pt; } return 0; @@ -3033,13 +3033,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, if (r) goto error_unreserve; + amdgpu_vm_bo_base_init(&vm->root.base, vm, root); + r = amdgpu_vm_clear_bo(adev, vm, root, adev->vm_manager.root_level, vm->pte_support_ats); if (r) goto error_unreserve; - amdgpu_vm_bo_base_init(&vm->root.base, vm, root); amdgpu_bo_unreserve(vm->root.base.bo); if (pasid) { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 5533f6e4f4a48..d0309e8c9d12c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -220,6 +220,7 @@ static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = { + SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) }; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c index c63de945c0214..0487e3a4e9e78 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c @@ -500,9 +500,7 @@ static bool psp_v3_1_smu_reload_quirk(struct psp_context *psp) struct amdgpu_device *adev = psp->adev; uint32_t reg; - reg = smnMP1_FIRMWARE_FLAGS | 0x03b00000; - WREG32_SOC15(NBIO, 0, mmPCIE_INDEX2, reg); - reg = RREG32_SOC15(NBIO, 0, mmPCIE_DATA2); + reg = RREG32_PCIE(smnMP1_FIRMWARE_FLAGS | 0x03b00000); return (reg & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) ? true : false; } diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index 99ebcf29dcb0f..ed89a101f73f3 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -461,7 +461,6 @@ static int soc15_asic_reset(struct amdgpu_device *adev) switch (adev->asic_type) { case CHIP_VEGA10: - case CHIP_VEGA20: soc15_asic_get_baco_capability(adev, &baco_reset); break; default: diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c index 47243165a082a..ae90a99909efe 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c @@ -323,57 +323,7 @@ static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, struct queue_properties *q) { - uint64_t addr; - struct cik_mqd *m; - int retval; - - retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), - mqd_mem_obj); - - if (retval != 0) - return -ENOMEM; - - m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; - addr = (*mqd_mem_obj)->gpu_addr; - - memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); - - m->header = 0xC0310800; - m->compute_pipelinestat_enable = 1; - m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; - m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; - m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; - m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; - - m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | - PRELOAD_REQ; - m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | - QUANTUM_DURATION(10); - - m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; - m->cp_mqd_base_addr_lo = lower_32_bits(addr); - m->cp_mqd_base_addr_hi = upper_32_bits(addr); - - m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; - - /* - * Pipe Priority - * Identifies the pipe relative priority when this queue is connected - * to the pipeline. The pipe priority is against the GFX pipe and HP3D. - * In KFD we are using a fixed pipe priority set to CS_MEDIUM. - * 0 = CS_LOW (typically below GFX) - * 1 = CS_MEDIUM (typically between HP3D and GFX - * 2 = CS_HIGH (typically above HP3D) - */ - m->cp_hqd_pipe_priority = 1; - m->cp_hqd_queue_priority = 15; - - *mqd = m; - if (gart_addr) - *gart_addr = addr; - retval = mm->update_mqd(mm, m, q); - - return retval; + return init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q); } static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 2f26581b93ff5..fb27783d7a542 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -886,6 +886,7 @@ static void emulated_link_detect(struct dc_link *link) return; } + /* dc_sink_create returns a new reference */ link->local_sink = sink; edid_status = dm_helpers_read_local_edid( @@ -952,6 +953,8 @@ static int dm_resume(void *handle) if (aconnector->fake_enable && aconnector->dc_link->local_sink) aconnector->fake_enable = false; + if (aconnector->dc_sink) + dc_sink_release(aconnector->dc_sink); aconnector->dc_sink = NULL; amdgpu_dm_update_connector_after_detect(aconnector); mutex_unlock(&aconnector->hpd_lock); @@ -1061,6 +1064,8 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) sink = aconnector->dc_link->local_sink; + if (sink) + dc_sink_retain(sink); /* * Edid mgmt connector gets first update only in mode_valid hook and then @@ -1085,21 +1090,24 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) * to it anymore after disconnect, so on next crtc to connector * reshuffle by UMD we will get into unwanted dc_sink release */ - if (aconnector->dc_sink != aconnector->dc_em_sink) - dc_sink_release(aconnector->dc_sink); + dc_sink_release(aconnector->dc_sink); } aconnector->dc_sink = sink; + dc_sink_retain(aconnector->dc_sink); amdgpu_dm_update_freesync_caps(connector, aconnector->edid); } else { amdgpu_dm_update_freesync_caps(connector, NULL); - if (!aconnector->dc_sink) + if (!aconnector->dc_sink) { aconnector->dc_sink = aconnector->dc_em_sink; - else if (aconnector->dc_sink != aconnector->dc_em_sink) dc_sink_retain(aconnector->dc_sink); + } } mutex_unlock(&dev->mode_config.mutex); + + if (sink) + dc_sink_release(sink); return; } @@ -1107,8 +1115,10 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) * TODO: temporary guard to look for proper fix * if this sink is MST sink, we should not do anything */ - if (sink && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT_MST) + if (sink && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { + dc_sink_release(sink); return; + } if (aconnector->dc_sink == sink) { /* @@ -1117,6 +1127,8 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) */ DRM_DEBUG_DRIVER("DCHPD: connector_id=%d: dc_sink didn't change.\n", aconnector->connector_id); + if (sink) + dc_sink_release(sink); return; } @@ -1138,6 +1150,7 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) amdgpu_dm_update_freesync_caps(connector, NULL); aconnector->dc_sink = sink; + dc_sink_retain(aconnector->dc_sink); if (sink->dc_edid.length == 0) { aconnector->edid = NULL; drm_dp_cec_unset_edid(&aconnector->dm_dp_aux.aux); @@ -1158,11 +1171,15 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector) amdgpu_dm_update_freesync_caps(connector, NULL); drm_connector_update_edid_property(connector, NULL); aconnector->num_modes = 0; + dc_sink_release(aconnector->dc_sink); aconnector->dc_sink = NULL; aconnector->edid = NULL; } mutex_unlock(&dev->mode_config.mutex); + + if (sink) + dc_sink_release(sink); } static void handle_hpd_irq(void *param) @@ -2977,6 +2994,7 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector, return stream; } else { sink = aconnector->dc_sink; + dc_sink_retain(sink); } stream = dc_create_stream_for_sink(sink); @@ -3042,8 +3060,7 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector, update_stream_signal(stream, sink); finish: - if (sink && sink->sink_signal == SIGNAL_TYPE_VIRTUAL && aconnector->base.force != DRM_FORCE_ON) - dc_sink_release(sink); + dc_sink_release(sink); return stream; } @@ -3301,6 +3318,14 @@ static void amdgpu_dm_connector_destroy(struct drm_connector *connector) dm->backlight_dev = NULL; } #endif + + if (aconnector->dc_em_sink) + dc_sink_release(aconnector->dc_em_sink); + aconnector->dc_em_sink = NULL; + if (aconnector->dc_sink) + dc_sink_release(aconnector->dc_sink); + aconnector->dc_sink = NULL; + drm_dp_cec_unregister_connector(&aconnector->dm_dp_aux.aux); drm_connector_unregister(connector); drm_connector_cleanup(connector); @@ -3398,10 +3423,12 @@ static void create_eml_sink(struct amdgpu_dm_connector *aconnector) (edid->extensions + 1) * EDID_LENGTH, &init_params); - if (aconnector->base.force == DRM_FORCE_ON) + if (aconnector->base.force == DRM_FORCE_ON) { aconnector->dc_sink = aconnector->dc_link->local_sink ? aconnector->dc_link->local_sink : aconnector->dc_em_sink; + dc_sink_retain(aconnector->dc_sink); + } } static void handle_edid_mgmt(struct amdgpu_dm_connector *aconnector) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index f51d52eb52e6d..c4ea3a91f17aa 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -191,6 +191,7 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector) &init_params); dc_sink->priv = aconnector; + /* dc_link_add_remote_sink returns a new reference */ aconnector->dc_sink = dc_sink; if (aconnector->dc_sink) diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c index 12d1842079ae5..eb62d10bb65cd 100644 --- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c +++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c @@ -1348,12 +1348,12 @@ void dcn_bw_update_from_pplib(struct dc *dc) struct dm_pp_clock_levels_with_voltage fclks = {0}, dcfclks = {0}; bool res; - kernel_fpu_begin(); - /* TODO: This is not the proper way to obtain fabric_and_dram_bandwidth, should be min(fclk, memclk) */ res = dm_pp_get_clock_levels_by_type_with_voltage( ctx, DM_PP_CLOCK_TYPE_FCLK, &fclks); + kernel_fpu_begin(); + if (res) res = verify_clock_values(&fclks); @@ -1372,9 +1372,13 @@ void dcn_bw_update_from_pplib(struct dc *dc) } else BREAK_TO_DEBUGGER(); + kernel_fpu_end(); + res = dm_pp_get_clock_levels_by_type_with_voltage( ctx, DM_PP_CLOCK_TYPE_DCFCLK, &dcfclks); + kernel_fpu_begin(); + if (res) res = verify_clock_values(&dcfclks); diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c index 7f5a947ad31df..4eba3c4800b63 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c @@ -794,6 +794,7 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason) sink->link->dongle_max_pix_clk = sink_caps.max_hdmi_pixel_clock; sink->converter_disable_audio = converter_disable_audio; + /* dc_sink_create returns a new reference */ link->local_sink = sink; edid_status = dm_helpers_read_local_edid( @@ -2037,6 +2038,9 @@ static enum dc_status enable_link( break; } + if (status == DC_OK) + pipe_ctx->stream->link->link_status.link_active = true; + return status; } @@ -2060,6 +2064,14 @@ static void disable_link(struct dc_link *link, enum signal_type signal) dp_disable_link_phy_mst(link, signal); } else link->link_enc->funcs->disable_output(link->link_enc, signal); + + if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { + /* MST disable link only when no stream use the link */ + if (link->mst_stream_alloc_table.stream_count <= 0) + link->link_status.link_active = false; + } else { + link->link_status.link_active = false; + } } static bool dp_active_dongle_validate_timing( @@ -2623,8 +2635,6 @@ void core_link_enable_stream( } } - stream->link->link_status.link_active = true; - core_dc->hwss.enable_audio_stream(pipe_ctx); /* turn off otg test pattern if enable */ @@ -2659,8 +2669,6 @@ void core_link_disable_stream(struct pipe_ctx *pipe_ctx, int option) core_dc->hwss.disable_stream(pipe_ctx, option); disable_link(pipe_ctx->stream->link, pipe_ctx->stream->signal); - - pipe_ctx->stream->link->link_status.link_active = false; } void core_link_set_avmute(struct pipe_ctx *pipe_ctx, bool enable) diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c index 94a84bc57c7a6..bfd27f10879e9 100644 --- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c +++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c @@ -724,7 +724,7 @@ static void build_vrr_infopacket_v1(enum signal_type signal, static void build_vrr_infopacket_v2(enum signal_type signal, const struct mod_vrr_params *vrr, - const enum color_transfer_func *app_tf, + enum color_transfer_func app_tf, struct dc_info_packet *infopacket) { unsigned int payload_size = 0; @@ -732,8 +732,7 @@ static void build_vrr_infopacket_v2(enum signal_type signal, build_vrr_infopacket_header_v2(signal, infopacket, &payload_size); build_vrr_infopacket_data(vrr, infopacket); - if (app_tf != NULL) - build_vrr_infopacket_fs2_data(*app_tf, infopacket); + build_vrr_infopacket_fs2_data(app_tf, infopacket); build_vrr_infopacket_checksum(&payload_size, infopacket); @@ -757,7 +756,7 @@ void mod_freesync_build_vrr_infopacket(struct mod_freesync *mod_freesync, const struct dc_stream_state *stream, const struct mod_vrr_params *vrr, enum vrr_packet_type packet_type, - const enum color_transfer_func *app_tf, + enum color_transfer_func app_tf, struct dc_info_packet *infopacket) { /* SPD info packet for FreeSync diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_freesync.h b/drivers/gpu/drm/amd/display/modules/inc/mod_freesync.h index 4222e403b1515..dcef85994c45d 100644 --- a/drivers/gpu/drm/amd/display/modules/inc/mod_freesync.h +++ b/drivers/gpu/drm/amd/display/modules/inc/mod_freesync.h @@ -145,7 +145,7 @@ void mod_freesync_build_vrr_infopacket(struct mod_freesync *mod_freesync, const struct dc_stream_state *stream, const struct mod_vrr_params *vrr, enum vrr_packet_type packet_type, - const enum color_transfer_func *app_tf, + enum color_transfer_func app_tf, struct dc_info_packet *infopacket); void mod_freesync_build_vrr_params(struct mod_freesync *mod_freesync, diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c index ce177d7f04cbe..6bf48934fdc4d 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c @@ -277,8 +277,7 @@ int psm_adjust_power_state_dynamic(struct pp_hwmgr *hwmgr, bool skip_display_set if (!skip_display_settings) phm_notify_smc_display_config_after_ps_adjustment(hwmgr); - if ((hwmgr->request_dpm_level != hwmgr->dpm_level) && - !phm_force_dpm_levels(hwmgr, hwmgr->request_dpm_level)) + if (!phm_force_dpm_levels(hwmgr, hwmgr->request_dpm_level)) hwmgr->dpm_level = hwmgr->request_dpm_level; if (hwmgr->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL) { diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c index 4588bddf8b332..615cf2c09e54e 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c @@ -489,15 +489,16 @@ int pp_atomfwctrl_get_gpio_information(struct pp_hwmgr *hwmgr, } int pp_atomfwctrl_get_clk_information_by_clkid(struct pp_hwmgr *hwmgr, - uint8_t id, uint32_t *frequency) + uint8_t clk_id, uint8_t syspll_id, + uint32_t *frequency) { struct amdgpu_device *adev = hwmgr->adev; struct atom_get_smu_clock_info_parameters_v3_1 parameters; struct atom_get_smu_clock_info_output_parameters_v3_1 *output; uint32_t ix; - parameters.clk_id = id; - parameters.syspll_id = 0; + parameters.clk_id = clk_id; + parameters.syspll_id = syspll_id; parameters.command = GET_SMU_CLOCK_INFO_V3_1_GET_CLOCK_FREQ; parameters.dfsdid = 0; @@ -530,20 +531,23 @@ static void pp_atomfwctrl_copy_vbios_bootup_values_3_2(struct pp_hwmgr *hwmgr, boot_values->ulSocClk = 0; boot_values->ulDCEFClk = 0; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_SOCCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_SOCCLK_ID, SMU11_SYSPLL0_ID, &frequency)) boot_values->ulSocClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_DCEFCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_DCEFCLK_ID, SMU11_SYSPLL0_ID, &frequency)) boot_values->ulDCEFClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_ECLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_ECLK_ID, SMU11_SYSPLL0_ID, &frequency)) boot_values->ulEClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_VCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_VCLK_ID, SMU11_SYSPLL0_ID, &frequency)) boot_values->ulVClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_DCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_DCLK_ID, SMU11_SYSPLL0_ID, &frequency)) boot_values->ulDClk = frequency; + + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL1_0_FCLK_ID, SMU11_SYSPLL1_2_ID, &frequency)) + boot_values->ulFClk = frequency; } static void pp_atomfwctrl_copy_vbios_bootup_values_3_1(struct pp_hwmgr *hwmgr, @@ -563,19 +567,19 @@ static void pp_atomfwctrl_copy_vbios_bootup_values_3_1(struct pp_hwmgr *hwmgr, boot_values->ulSocClk = 0; boot_values->ulDCEFClk = 0; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_SOCCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_SOCCLK_ID, 0, &frequency)) boot_values->ulSocClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_DCEFCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_DCEFCLK_ID, 0, &frequency)) boot_values->ulDCEFClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_ECLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_ECLK_ID, 0, &frequency)) boot_values->ulEClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_VCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_VCLK_ID, 0, &frequency)) boot_values->ulVClk = frequency; - if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_DCLK_ID, &frequency)) + if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_DCLK_ID, 0, &frequency)) boot_values->ulDClk = frequency; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h index fe9e8ceef50e6..b7e2651b570bc 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h @@ -139,6 +139,7 @@ struct pp_atomfwctrl_bios_boot_up_values { uint32_t ulEClk; uint32_t ulVClk; uint32_t ulDClk; + uint32_t ulFClk; uint16_t usVddc; uint16_t usVddci; uint16_t usMvddc; @@ -236,7 +237,8 @@ int pp_atomfwctrl_get_vbios_bootup_values(struct pp_hwmgr *hwmgr, int pp_atomfwctrl_get_smc_dpm_information(struct pp_hwmgr *hwmgr, struct pp_atomfwctrl_smc_dpm_parameters *param); int pp_atomfwctrl_get_clk_information_by_clkid(struct pp_hwmgr *hwmgr, - uint8_t id, uint32_t *frequency); + uint8_t clk_id, uint8_t syspll_id, + uint32_t *frequency); #endif diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c index 48187acac59e7..83d3d935f3acc 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c @@ -3491,14 +3491,14 @@ static int smu7_get_gpu_power(struct pp_hwmgr *hwmgr, u32 *query) smum_send_msg_to_smc(hwmgr, PPSMC_MSG_PmStatusLogStart); cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, - ixSMU_PM_STATUS_94, 0); + ixSMU_PM_STATUS_95, 0); for (i = 0; i < 10; i++) { - mdelay(1); + mdelay(500); smum_send_msg_to_smc(hwmgr, PPSMC_MSG_PmStatusLogSample); tmp = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, - ixSMU_PM_STATUS_94); + ixSMU_PM_STATUS_95); if (tmp != 0) break; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c index 5479125ff4f6e..5c4f701939ea5 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c @@ -2575,10 +2575,10 @@ static int vega10_init_smc_table(struct pp_hwmgr *hwmgr) data->vbios_boot_state.gfx_clock = boot_up_values.ulGfxClk; data->vbios_boot_state.mem_clock = boot_up_values.ulUClk; pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, - SMU9_SYSPLL0_SOCCLK_ID, &boot_up_values.ulSocClk); + SMU9_SYSPLL0_SOCCLK_ID, 0, &boot_up_values.ulSocClk); pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, - SMU9_SYSPLL0_DCEFCLK_ID, &boot_up_values.ulDCEFClk); + SMU9_SYSPLL0_DCEFCLK_ID, 0, &boot_up_values.ulDCEFClk); data->vbios_boot_state.soc_clock = boot_up_values.ulSocClk; data->vbios_boot_state.dcef_clock = boot_up_values.ulDCEFClk; @@ -4407,9 +4407,9 @@ static int vega10_set_ppfeature_status(struct pp_hwmgr *hwmgr, uint64_t new_ppfe return ret; features_to_disable = - (features_enabled ^ new_ppfeature_masks) & features_enabled; + features_enabled & ~new_ppfeature_masks; features_to_enable = - (features_enabled ^ new_ppfeature_masks) ^ features_to_disable; + ~features_enabled & new_ppfeature_masks; pr_debug("features_to_disable 0x%llx\n", features_to_disable); pr_debug("features_to_enable 0x%llx\n", features_to_enable); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c index 6c8e78611c033..bdb48e94eff60 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c @@ -2009,9 +2009,9 @@ static int vega12_set_ppfeature_status(struct pp_hwmgr *hwmgr, uint64_t new_ppfe return ret; features_to_disable = - (features_enabled ^ new_ppfeature_masks) & features_enabled; + features_enabled & ~new_ppfeature_masks; features_to_enable = - (features_enabled ^ new_ppfeature_masks) ^ features_to_disable; + ~features_enabled & new_ppfeature_masks; pr_debug("features_to_disable 0x%llx\n", features_to_disable); pr_debug("features_to_enable 0x%llx\n", features_to_enable); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c index aad79affb0812..9aa7bec1b5fe6 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c @@ -463,9 +463,9 @@ static int vega20_setup_asic_task(struct pp_hwmgr *hwmgr) static void vega20_init_dpm_state(struct vega20_dpm_state *dpm_state) { dpm_state->soft_min_level = 0x0; - dpm_state->soft_max_level = 0xffff; + dpm_state->soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_state->hard_min_level = 0x0; - dpm_state->hard_max_level = 0xffff; + dpm_state->hard_max_level = VG20_CLOCK_MAX_DEFAULT; } static int vega20_get_number_of_dpm_level(struct pp_hwmgr *hwmgr, @@ -711,8 +711,10 @@ static int vega20_setup_default_dpm_tables(struct pp_hwmgr *hwmgr) PP_ASSERT_WITH_CODE(!ret, "[SetupDefaultDpmTable] failed to get fclk dpm levels!", return ret); - } else - dpm_table->count = 0; + } else { + dpm_table->count = 1; + dpm_table->dpm_levels[0].value = data->vbios_boot_state.fclock / 100; + } vega20_init_dpm_state(&(dpm_table->dpm_state)); /* save a copy of the default DPM table */ @@ -754,6 +756,7 @@ static int vega20_init_smc_table(struct pp_hwmgr *hwmgr) data->vbios_boot_state.eclock = boot_up_values.ulEClk; data->vbios_boot_state.vclock = boot_up_values.ulVClk; data->vbios_boot_state.dclock = boot_up_values.ulDClk; + data->vbios_boot_state.fclock = boot_up_values.ulFClk; data->vbios_boot_state.uc_cooling_id = boot_up_values.ucCoolingID; smum_send_msg_to_smc_with_parameter(hwmgr, @@ -780,6 +783,8 @@ static int vega20_init_smc_table(struct pp_hwmgr *hwmgr) static int vega20_override_pcie_parameters(struct pp_hwmgr *hwmgr) { struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev); + struct vega20_hwmgr *data = + (struct vega20_hwmgr *)(hwmgr->backend); uint32_t pcie_gen = 0, pcie_width = 0, smu_pcie_arg; int ret; @@ -816,6 +821,10 @@ static int vega20_override_pcie_parameters(struct pp_hwmgr *hwmgr) "[OverridePcieParameters] Attempt to override pcie params failed!", return ret); + data->pcie_parameters_override = 1; + data->pcie_gen_level1 = pcie_gen; + data->pcie_width_level1 = pcie_width; + return 0; } @@ -979,6 +988,8 @@ static int vega20_od8_set_feature_capabilities( } if (data->smu_features[GNLD_DPM_UCLK].enabled) { + pptable_information->od_settings_min[OD8_SETTING_UCLK_FMAX] = + data->dpm_table.mem_table.dpm_levels[data->dpm_table.mem_table.count - 2].value; if (pptable_information->od_feature_capabilities[ATOM_VEGA20_ODFEATURE_UCLK_MAX] && pptable_information->od_settings_min[OD8_SETTING_UCLK_FMAX] > 0 && pptable_information->od_settings_max[OD8_SETTING_UCLK_FMAX] > 0 && @@ -2314,32 +2325,8 @@ static int vega20_force_dpm_lowest(struct pp_hwmgr *hwmgr) static int vega20_unforce_dpm_levels(struct pp_hwmgr *hwmgr) { - struct vega20_hwmgr *data = - (struct vega20_hwmgr *)(hwmgr->backend); - uint32_t soft_min_level, soft_max_level; int ret = 0; - soft_min_level = vega20_find_lowest_dpm_level(&(data->dpm_table.gfx_table)); - soft_max_level = vega20_find_highest_dpm_level(&(data->dpm_table.gfx_table)); - data->dpm_table.gfx_table.dpm_state.soft_min_level = - data->dpm_table.gfx_table.dpm_levels[soft_min_level].value; - data->dpm_table.gfx_table.dpm_state.soft_max_level = - data->dpm_table.gfx_table.dpm_levels[soft_max_level].value; - - soft_min_level = vega20_find_lowest_dpm_level(&(data->dpm_table.mem_table)); - soft_max_level = vega20_find_highest_dpm_level(&(data->dpm_table.mem_table)); - data->dpm_table.mem_table.dpm_state.soft_min_level = - data->dpm_table.mem_table.dpm_levels[soft_min_level].value; - data->dpm_table.mem_table.dpm_state.soft_max_level = - data->dpm_table.mem_table.dpm_levels[soft_max_level].value; - - soft_min_level = vega20_find_lowest_dpm_level(&(data->dpm_table.soc_table)); - soft_max_level = vega20_find_highest_dpm_level(&(data->dpm_table.soc_table)); - data->dpm_table.soc_table.dpm_state.soft_min_level = - data->dpm_table.soc_table.dpm_levels[soft_min_level].value; - data->dpm_table.soc_table.dpm_state.soft_max_level = - data->dpm_table.soc_table.dpm_levels[soft_max_level].value; - ret = vega20_upload_dpm_min_level(hwmgr, 0xFFFFFFFF); PP_ASSERT_WITH_CODE(!ret, "Failed to upload DPM Bootup Levels!", @@ -2641,9 +2628,8 @@ static int vega20_get_sclks(struct pp_hwmgr *hwmgr, struct vega20_single_dpm_table *dpm_table = &(data->dpm_table.gfx_table); int i, count; - PP_ASSERT_WITH_CODE(data->smu_features[GNLD_DPM_GFXCLK].enabled, - "[GetSclks]: gfxclk dpm not enabled!\n", - return -EPERM); + if (!data->smu_features[GNLD_DPM_GFXCLK].enabled) + return -1; count = (dpm_table->count > MAX_NUM_CLOCKS) ? MAX_NUM_CLOCKS : dpm_table->count; clocks->num_levels = count; @@ -2670,9 +2656,8 @@ static int vega20_get_memclocks(struct pp_hwmgr *hwmgr, struct vega20_single_dpm_table *dpm_table = &(data->dpm_table.mem_table); int i, count; - PP_ASSERT_WITH_CODE(data->smu_features[GNLD_DPM_UCLK].enabled, - "[GetMclks]: uclk dpm not enabled!\n", - return -EPERM); + if (!data->smu_features[GNLD_DPM_UCLK].enabled) + return -1; count = (dpm_table->count > MAX_NUM_CLOCKS) ? MAX_NUM_CLOCKS : dpm_table->count; clocks->num_levels = data->mclk_latency_table.count = count; @@ -2696,9 +2681,8 @@ static int vega20_get_dcefclocks(struct pp_hwmgr *hwmgr, struct vega20_single_dpm_table *dpm_table = &(data->dpm_table.dcef_table); int i, count; - PP_ASSERT_WITH_CODE(data->smu_features[GNLD_DPM_DCEFCLK].enabled, - "[GetDcfclocks]: dcefclk dpm not enabled!\n", - return -EPERM); + if (!data->smu_features[GNLD_DPM_DCEFCLK].enabled) + return -1; count = (dpm_table->count > MAX_NUM_CLOCKS) ? MAX_NUM_CLOCKS : dpm_table->count; clocks->num_levels = count; @@ -2719,9 +2703,8 @@ static int vega20_get_socclocks(struct pp_hwmgr *hwmgr, struct vega20_single_dpm_table *dpm_table = &(data->dpm_table.soc_table); int i, count; - PP_ASSERT_WITH_CODE(data->smu_features[GNLD_DPM_SOCCLK].enabled, - "[GetSocclks]: socclk dpm not enabled!\n", - return -EPERM); + if (!data->smu_features[GNLD_DPM_SOCCLK].enabled) + return -1; count = (dpm_table->count > MAX_NUM_CLOCKS) ? MAX_NUM_CLOCKS : dpm_table->count; clocks->num_levels = count; @@ -2799,7 +2782,6 @@ static int vega20_odn_edit_dpm_table(struct pp_hwmgr *hwmgr, data->od8_settings.od8_settings_array; OverDriveTable_t *od_table = &(data->smc_state_table.overdrive_table); - struct pp_clock_levels_with_latency clocks; int32_t input_index, input_clk, input_vol, i; int od8_id; int ret; @@ -2858,11 +2840,6 @@ static int vega20_odn_edit_dpm_table(struct pp_hwmgr *hwmgr, return -EOPNOTSUPP; } - ret = vega20_get_memclocks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Attempt to get memory clk levels failed!", - return ret); - for (i = 0; i < size; i += 2) { if (i + 2 > size) { pr_info("invalid number of input parameters %d\n", @@ -2879,11 +2856,11 @@ static int vega20_odn_edit_dpm_table(struct pp_hwmgr *hwmgr, return -EINVAL; } - if (input_clk < clocks.data[0].clocks_in_khz / 1000 || + if (input_clk < od8_settings[OD8_SETTING_UCLK_FMAX].min_value || input_clk > od8_settings[OD8_SETTING_UCLK_FMAX].max_value) { pr_info("clock freq %d is not within allowed range [%d - %d]\n", input_clk, - clocks.data[0].clocks_in_khz / 1000, + od8_settings[OD8_SETTING_UCLK_FMAX].min_value, od8_settings[OD8_SETTING_UCLK_FMAX].max_value); return -EINVAL; } @@ -3088,9 +3065,9 @@ static int vega20_set_ppfeature_status(struct pp_hwmgr *hwmgr, uint64_t new_ppfe return ret; features_to_disable = - (features_enabled ^ new_ppfeature_masks) & features_enabled; + features_enabled & ~new_ppfeature_masks; features_to_enable = - (features_enabled ^ new_ppfeature_masks) ^ features_to_disable; + ~features_enabled & new_ppfeature_masks; pr_debug("features_to_disable 0x%llx\n", features_to_disable); pr_debug("features_to_enable 0x%llx\n", features_to_enable); @@ -3128,7 +3105,7 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, &(data->dpm_table.fclk_table); int i, now, size = 0; int ret = 0; - uint32_t gen_speed, lane_width; + uint32_t gen_speed, lane_width, current_gen_speed, current_lane_width; switch (type) { case PP_SCLK: @@ -3137,10 +3114,11 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get current gfx clk Failed!", return ret); - ret = vega20_get_sclks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Attempt to get gfx clk levels Failed!", - return ret); + if (vega20_get_sclks(hwmgr, &clocks)) { + size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + now / 100); + break; + } for (i = 0; i < clocks.num_levels; i++) size += sprintf(buf + size, "%d: %uMhz %s\n", @@ -3154,10 +3132,11 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get current mclk freq Failed!", return ret); - ret = vega20_get_memclocks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Attempt to get memory clk levels Failed!", - return ret); + if (vega20_get_memclocks(hwmgr, &clocks)) { + size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + now / 100); + break; + } for (i = 0; i < clocks.num_levels; i++) size += sprintf(buf + size, "%d: %uMhz %s\n", @@ -3171,10 +3150,11 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get current socclk freq Failed!", return ret); - ret = vega20_get_socclocks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Attempt to get soc clk levels Failed!", - return ret); + if (vega20_get_socclocks(hwmgr, &clocks)) { + size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + now / 100); + break; + } for (i = 0; i < clocks.num_levels; i++) size += sprintf(buf + size, "%d: %uMhz %s\n", @@ -3200,10 +3180,11 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, "Attempt to get current dcefclk freq Failed!", return ret); - ret = vega20_get_dcefclocks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Attempt to get dcefclk levels Failed!", - return ret); + if (vega20_get_dcefclocks(hwmgr, &clocks)) { + size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n", + now / 100); + break; + } for (i = 0; i < clocks.num_levels; i++) size += sprintf(buf + size, "%d: %uMhz %s\n", @@ -3212,28 +3193,36 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, break; case PP_PCIE: - gen_speed = (RREG32_PCIE(smnPCIE_LC_SPEED_CNTL) & + current_gen_speed = (RREG32_PCIE(smnPCIE_LC_SPEED_CNTL) & PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >> PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT; - lane_width = (RREG32_PCIE(smnPCIE_LC_LINK_WIDTH_CNTL) & + current_lane_width = (RREG32_PCIE(smnPCIE_LC_LINK_WIDTH_CNTL) & PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK) >> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT; - for (i = 0; i < NUM_LINK_LEVELS; i++) + for (i = 0; i < NUM_LINK_LEVELS; i++) { + if (i == 1 && data->pcie_parameters_override) { + gen_speed = data->pcie_gen_level1; + lane_width = data->pcie_width_level1; + } else { + gen_speed = pptable->PcieGenSpeed[i]; + lane_width = pptable->PcieLaneCount[i]; + } size += sprintf(buf + size, "%d: %s %s %dMhz %s\n", i, - (pptable->PcieGenSpeed[i] == 0) ? "2.5GT/s," : - (pptable->PcieGenSpeed[i] == 1) ? "5.0GT/s," : - (pptable->PcieGenSpeed[i] == 2) ? "8.0GT/s," : - (pptable->PcieGenSpeed[i] == 3) ? "16.0GT/s," : "", - (pptable->PcieLaneCount[i] == 1) ? "x1" : - (pptable->PcieLaneCount[i] == 2) ? "x2" : - (pptable->PcieLaneCount[i] == 3) ? "x4" : - (pptable->PcieLaneCount[i] == 4) ? "x8" : - (pptable->PcieLaneCount[i] == 5) ? "x12" : - (pptable->PcieLaneCount[i] == 6) ? "x16" : "", + (gen_speed == 0) ? "2.5GT/s," : + (gen_speed == 1) ? "5.0GT/s," : + (gen_speed == 2) ? "8.0GT/s," : + (gen_speed == 3) ? "16.0GT/s," : "", + (lane_width == 1) ? "x1" : + (lane_width == 2) ? "x2" : + (lane_width == 3) ? "x4" : + (lane_width == 4) ? "x8" : + (lane_width == 5) ? "x12" : + (lane_width == 6) ? "x16" : "", pptable->LclkFreq[i], - (gen_speed == pptable->PcieGenSpeed[i]) && - (lane_width == pptable->PcieLaneCount[i]) ? + (current_gen_speed == gen_speed) && + (current_lane_width == lane_width) ? "*" : ""); + } break; case OD_SCLK: @@ -3288,13 +3277,8 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr, } if (od8_settings[OD8_SETTING_UCLK_FMAX].feature_id) { - ret = vega20_get_memclocks(hwmgr, &clocks); - PP_ASSERT_WITH_CODE(!ret, - "Fail to get memory clk levels!", - return ret); - size += sprintf(buf + size, "MCLK: %7uMhz %10uMhz\n", - clocks.data[0].clocks_in_khz / 1000, + od8_settings[OD8_SETTING_UCLK_FMAX].min_value, od8_settings[OD8_SETTING_UCLK_FMAX].max_value); } @@ -3356,6 +3340,31 @@ static int vega20_set_uclk_to_highest_dpm_level(struct pp_hwmgr *hwmgr, return ret; } +static int vega20_set_fclk_to_highest_dpm_level(struct pp_hwmgr *hwmgr) +{ + struct vega20_hwmgr *data = (struct vega20_hwmgr *)(hwmgr->backend); + struct vega20_single_dpm_table *dpm_table = &(data->dpm_table.fclk_table); + int ret = 0; + + if (data->smu_features[GNLD_DPM_FCLK].enabled) { + PP_ASSERT_WITH_CODE(dpm_table->count > 0, + "[SetFclkToHightestDpmLevel] Dpm table has no entry!", + return -EINVAL); + PP_ASSERT_WITH_CODE(dpm_table->count <= NUM_FCLK_DPM_LEVELS, + "[SetFclkToHightestDpmLevel] Dpm table has too many entries!", + return -EINVAL); + + dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(hwmgr, + PPSMC_MSG_SetSoftMinByFreq, + (PPCLK_FCLK << 16 ) | dpm_table->dpm_state.soft_min_level)), + "[SetFclkToHightestDpmLevel] Set soft min fclk failed!", + return ret); + } + + return ret; +} + static int vega20_pre_display_configuration_changed_task(struct pp_hwmgr *hwmgr) { struct vega20_hwmgr *data = (struct vega20_hwmgr *)(hwmgr->backend); @@ -3366,8 +3375,10 @@ static int vega20_pre_display_configuration_changed_task(struct pp_hwmgr *hwmgr) ret = vega20_set_uclk_to_highest_dpm_level(hwmgr, &data->dpm_table.mem_table); + if (ret) + return ret; - return ret; + return vega20_set_fclk_to_highest_dpm_level(hwmgr); } static int vega20_display_configuration_changed_task(struct pp_hwmgr *hwmgr) @@ -3461,9 +3472,9 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) /* gfxclk */ dpm_table = &(data->dpm_table.gfx_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_GFXCLK_LEVEL < dpm_table->count) { @@ -3485,9 +3496,9 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) /* memclk */ dpm_table = &(data->dpm_table.mem_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_MCLK_LEVEL < dpm_table->count) { @@ -3526,12 +3537,21 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) if (hwmgr->display_config->nb_pstate_switch_disable) dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + /* fclk */ + dpm_table = &(data->dpm_table.fclk_table); + dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; + dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; + if (hwmgr->display_config->nb_pstate_switch_disable) + dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + /* vclk */ dpm_table = &(data->dpm_table.vclk_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_UVDCLK_LEVEL < dpm_table->count) { @@ -3548,9 +3568,9 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) /* dclk */ dpm_table = &(data->dpm_table.dclk_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_UVDCLK_LEVEL < dpm_table->count) { @@ -3567,9 +3587,9 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) /* socclk */ dpm_table = &(data->dpm_table.soc_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_SOCCLK_LEVEL < dpm_table->count) { @@ -3586,9 +3606,9 @@ static int vega20_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr) /* eclk */ dpm_table = &(data->dpm_table.eclk_table); dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.soft_max_level = VG20_CLOCK_MAX_DEFAULT; dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value; - dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value; + dpm_table->dpm_state.hard_max_level = VG20_CLOCK_MAX_DEFAULT; if (PP_CAP(PHM_PlatformCaps_UMDPState)) { if (VEGA20_UMD_PSTATE_VCEMCLK_LEVEL < dpm_table->count) { diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h index 37f5f5e657da7..a5bc758ae0972 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h @@ -42,6 +42,8 @@ #define AVFS_CURVE 0 #define OD8_HOTCURVE_TEMPERATURE 85 +#define VG20_CLOCK_MAX_DEFAULT 0xFFFF + typedef uint32_t PP_Clock; enum { @@ -219,6 +221,7 @@ struct vega20_vbios_boot_state { uint32_t eclock; uint32_t dclock; uint32_t vclock; + uint32_t fclock; }; #define DPMTABLE_OD_UPDATE_SCLK 0x00000001 @@ -523,6 +526,10 @@ struct vega20_hwmgr { unsigned long metrics_time; SmuMetrics_t metrics_table; + + bool pcie_parameters_override; + uint32_t pcie_gen_level1; + uint32_t pcie_width_level1; }; #define VEGA20_DPM2_NEAR_TDP_DEC 10 diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_processpptables.c index 97f8a1a970c37..7a7f15d0c53af 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_processpptables.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_processpptables.c @@ -32,6 +32,8 @@ #include "cgs_common.h" #include "vega20_pptable.h" +#define VEGA20_FAN_TARGET_TEMPERATURE_OVERRIDE 105 + static void set_hw_cap(struct pp_hwmgr *hwmgr, bool enable, enum phm_platform_caps cap) { @@ -798,6 +800,17 @@ static int append_vbios_pptable(struct pp_hwmgr *hwmgr, PPTable_t *ppsmc_pptable return 0; } +static int override_powerplay_table_fantargettemperature(struct pp_hwmgr *hwmgr) +{ + struct phm_ppt_v3_information *pptable_information = + (struct phm_ppt_v3_information *)hwmgr->pptable; + PPTable_t *ppsmc_pptable = (PPTable_t *)(pptable_information->smc_pptable); + + ppsmc_pptable->FanTargetTemperature = VEGA20_FAN_TARGET_TEMPERATURE_OVERRIDE; + + return 0; +} + #define VEGA20_ENGINECLOCK_HARDMAX 198000 static int init_powerplay_table_information( struct pp_hwmgr *hwmgr, @@ -887,6 +900,10 @@ static int init_powerplay_table_information( result = append_vbios_pptable(hwmgr, (pptable_information->smc_pptable)); + if (result) + return result; + + result = override_powerplay_table_fantargettemperature(hwmgr); return result; } diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c index 52abca065764a..2d4cfe14f72e0 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c @@ -2330,6 +2330,7 @@ static uint32_t polaris10_get_offsetof(uint32_t type, uint32_t member) case DRAM_LOG_BUFF_SIZE: return offsetof(SMU74_SoftRegisters, DRAM_LOG_BUFF_SIZE); } + break; case SMU_Discrete_DpmTable: switch (member) { case UvdBootLevel: @@ -2339,6 +2340,7 @@ static uint32_t polaris10_get_offsetof(uint32_t type, uint32_t member) case LowSclkInterruptThreshold: return offsetof(SMU74_Discrete_DpmTable, LowSclkInterruptThreshold); } + break; } pr_warn("can't get the offset of type %x member %x\n", type, member); return 0; diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c index 079fc8e8f709f..742b3dc1f6cba 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c @@ -40,10 +40,8 @@ bool smu9_is_smc_ram_running(struct pp_hwmgr *hwmgr) struct amdgpu_device *adev = hwmgr->adev; uint32_t mp1_fw_flags; - WREG32_SOC15(NBIF, 0, mmPCIE_INDEX2, - (MP1_Public | (smnMP1_FIRMWARE_FLAGS & 0xffffffff))); - - mp1_fw_flags = RREG32_SOC15(NBIF, 0, mmPCIE_DATA2); + mp1_fw_flags = RREG32_PCIE(MP1_Public | + (smnMP1_FIRMWARE_FLAGS & 0xffffffff)); if (mp1_fw_flags & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) return true; diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c index b7ff7d4d6f448..ba00744c3413f 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c @@ -49,10 +49,8 @@ static bool vega20_is_smc_ram_running(struct pp_hwmgr *hwmgr) struct amdgpu_device *adev = hwmgr->adev; uint32_t mp1_fw_flags; - WREG32_SOC15(NBIF, 0, mmPCIE_INDEX2, - (MP1_Public | (smnMP1_FIRMWARE_FLAGS & 0xffffffff))); - - mp1_fw_flags = RREG32_SOC15(NBIF, 0, mmPCIE_DATA2); + mp1_fw_flags = RREG32_PCIE(MP1_Public | + (smnMP1_FIRMWARE_FLAGS & 0xffffffff)); if ((mp1_fw_flags & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) >> MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT) diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 540a77a2ade9d..40ac198480345 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -3039,9 +3039,31 @@ commit: return 0; } -static int __drm_atomic_helper_disable_all(struct drm_device *dev, - struct drm_modeset_acquire_ctx *ctx, - bool clean_old_fbs) +/** + * drm_atomic_helper_disable_all - disable all currently active outputs + * @dev: DRM device + * @ctx: lock acquisition context + * + * Loops through all connectors, finding those that aren't turned off and then + * turns them off by setting their DPMS mode to OFF and deactivating the CRTC + * that they are connected to. + * + * This is used for example in suspend/resume to disable all currently active + * functions when suspending. If you just want to shut down everything at e.g. + * driver unload, look at drm_atomic_helper_shutdown(). + * + * Note that if callers haven't already acquired all modeset locks this might + * return -EDEADLK, which must be handled by calling drm_modeset_backoff(). + * + * Returns: + * 0 on success or a negative error code on failure. + * + * See also: + * drm_atomic_helper_suspend(), drm_atomic_helper_resume() and + * drm_atomic_helper_shutdown(). + */ +int drm_atomic_helper_disable_all(struct drm_device *dev, + struct drm_modeset_acquire_ctx *ctx) { struct drm_atomic_state *state; struct drm_connector_state *conn_state; @@ -3099,35 +3121,6 @@ free: drm_atomic_state_put(state); return ret; } - -/** - * drm_atomic_helper_disable_all - disable all currently active outputs - * @dev: DRM device - * @ctx: lock acquisition context - * - * Loops through all connectors, finding those that aren't turned off and then - * turns them off by setting their DPMS mode to OFF and deactivating the CRTC - * that they are connected to. - * - * This is used for example in suspend/resume to disable all currently active - * functions when suspending. If you just want to shut down everything at e.g. - * driver unload, look at drm_atomic_helper_shutdown(). - * - * Note that if callers haven't already acquired all modeset locks this might - * return -EDEADLK, which must be handled by calling drm_modeset_backoff(). - * - * Returns: - * 0 on success or a negative error code on failure. - * - * See also: - * drm_atomic_helper_suspend(), drm_atomic_helper_resume() and - * drm_atomic_helper_shutdown(). - */ -int drm_atomic_helper_disable_all(struct drm_device *dev, - struct drm_modeset_acquire_ctx *ctx) -{ - return __drm_atomic_helper_disable_all(dev, ctx, false); -} EXPORT_SYMBOL(drm_atomic_helper_disable_all); /** @@ -3148,7 +3141,7 @@ void drm_atomic_helper_shutdown(struct drm_device *dev) DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); - ret = __drm_atomic_helper_disable_all(dev, &ctx, true); + ret = drm_atomic_helper_disable_all(dev, &ctx); if (ret) DRM_ERROR("Disabling all crtc's during unload failed with %i\n", ret); diff --git a/drivers/gpu/drm/drm_ioc32.c b/drivers/gpu/drm/drm_ioc32.c index 67b1fca39aa6c..0e3043e08c694 100644 --- a/drivers/gpu/drm/drm_ioc32.c +++ b/drivers/gpu/drm/drm_ioc32.c @@ -185,7 +185,7 @@ static int compat_drm_getmap(struct file *file, unsigned int cmd, m32.size = map.size; m32.type = map.type; m32.flags = map.flags; - m32.handle = ptr_to_compat(map.handle); + m32.handle = ptr_to_compat((void __user *)map.handle); m32.mtrr = map.mtrr; if (copy_to_user(argp, &m32, sizeof(m32))) return -EFAULT; @@ -216,7 +216,7 @@ static int compat_drm_addmap(struct file *file, unsigned int cmd, m32.offset = map.offset; m32.mtrr = map.mtrr; - m32.handle = ptr_to_compat(map.handle); + m32.handle = ptr_to_compat((void __user *)map.handle); if (map.handle != compat_ptr(m32.handle)) pr_err_ratelimited("compat_drm_addmap truncated handle %p for type %d offset %x\n", map.handle, m32.type, m32.offset); @@ -526,7 +526,7 @@ static int compat_drm_getsareactx(struct file *file, unsigned int cmd, if (err) return err; - req32.handle = ptr_to_compat(req.handle); + req32.handle = ptr_to_compat((void __user *)req.handle); if (copy_to_user(argp, &req32, sizeof(req32))) return -EFAULT; diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig index 041a77e400d4e..21df44b78df3c 100644 --- a/drivers/gpu/drm/etnaviv/Kconfig +++ b/drivers/gpu/drm/etnaviv/Kconfig @@ -2,7 +2,6 @@ config DRM_ETNAVIV tristate "ETNAVIV (DRM support for Vivante GPU IP cores)" depends on DRM - depends on ARCH_MXC || ARCH_DOVE || (ARM && COMPILE_TEST) depends on MMU select SHMEM select SYNC_FILE diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h index acb68c6983638..4d5d1a77eb2ab 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h @@ -15,8 +15,6 @@ struct etnaviv_perfmon_request; struct etnaviv_cmdbuf { /* suballocator this cmdbuf is allocated from */ struct etnaviv_cmdbuf_suballoc *suballoc; - /* user context key, must be unique between all active users */ - struct etnaviv_file_private *ctx; /* cmdbuf properties */ int suballoc_offset; void *vaddr; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c index 3fbb4855396cc..33854c94cb858 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c @@ -215,7 +215,7 @@ void etnaviv_core_dump(struct etnaviv_gpu *gpu) mutex_lock(&obj->lock); pages = etnaviv_gem_get_pages(obj); mutex_unlock(&obj->lock); - if (pages) { + if (!IS_ERR(pages)) { int j; iter.hdr->data[0] = bomap - bomap_start; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.h b/drivers/gpu/drm/etnaviv/etnaviv_gem.h index 76079c2291f88..f0abb744ef955 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.h @@ -95,6 +95,7 @@ struct etnaviv_gem_submit_bo { struct etnaviv_gem_submit { struct drm_sched_job sched_job; struct kref refcount; + struct etnaviv_file_private *ctx; struct etnaviv_gpu *gpu; struct dma_fence *out_fence, *in_fence; int out_fence_id; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c index 0566171f8df22..f21529e635e3d 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c @@ -15,7 +15,7 @@ struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj) int npages = obj->size >> PAGE_SHIFT; if (WARN_ON(!etnaviv_obj->pages)) /* should have already pinned! */ - return NULL; + return ERR_PTR(-EINVAL); return drm_prime_pages_to_sg(etnaviv_obj->pages, npages); } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c index 30875f8f29337..b2fe3446bfbcd 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c @@ -506,7 +506,7 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, if (ret) goto err_submit_objects; - submit->cmdbuf.ctx = file->driver_priv; + submit->ctx = file->driver_priv; submit->exec_state = args->exec_state; submit->flags = args->flags; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c index f1c88d8ad5ba8..f794e04be9e67 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c @@ -320,8 +320,8 @@ etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu) domain = &etnaviv_domain->base; domain->dev = gpu->dev; - domain->base = 0; - domain->size = (u64)SZ_1G * 4; + domain->base = SZ_4K; + domain->size = (u64)SZ_1G * 4 - SZ_4K; domain->ops = &etnaviv_iommuv2_ops; ret = etnaviv_iommuv2_init(etnaviv_domain); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c index 9980d81a26e3c..4227a4006c349 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c @@ -113,7 +113,7 @@ static const struct etnaviv_pm_domain doms_3d[] = { .name = "PE", .profile_read = VIVS_MC_PROFILE_PE_READ, .profile_config = VIVS_MC_PROFILE_CONFIG0, - .nr_signals = 5, + .nr_signals = 4, .signal = (const struct etnaviv_pm_signal[]) { { "PIXEL_COUNT_KILLED_BY_COLOR_PIPE", @@ -435,7 +435,7 @@ int etnaviv_pm_query_sig(struct etnaviv_gpu *gpu, dom = meta->domains + signal->domain; - if (signal->iter > dom->nr_signals) + if (signal->iter >= dom->nr_signals) return -EINVAL; sig = &dom->signal[signal->iter]; @@ -461,7 +461,7 @@ int etnaviv_pm_req_validate(const struct drm_etnaviv_gem_submit_pmr *r, dom = meta->domains + r->domain; - if (r->signal > dom->nr_signals) + if (r->signal >= dom->nr_signals) return -EINVAL; return 0; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 67ae266020244..6d24fea1766b0 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -153,7 +153,7 @@ int etnaviv_sched_push_job(struct drm_sched_entity *sched_entity, mutex_lock(&submit->gpu->fence_lock); ret = drm_sched_job_init(&submit->sched_job, sched_entity, - submit->cmdbuf.ctx); + submit->ctx); if (ret) goto out_unlock; diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c index 215b6ff8aa730..db7bb5bd5adde 100644 --- a/drivers/gpu/drm/i915/i915_active.c +++ b/drivers/gpu/drm/i915/i915_active.c @@ -163,17 +163,25 @@ int i915_active_ref(struct i915_active *ref, struct i915_request *rq) { struct i915_active_request *active; + int err = 0; + + /* Prevent reaping in case we malloc/wait while building the tree */ + i915_active_acquire(ref); active = active_instance(ref, timeline); - if (IS_ERR(active)) - return PTR_ERR(active); + if (IS_ERR(active)) { + err = PTR_ERR(active); + goto out; + } if (!i915_active_request_isset(active)) ref->count++; __i915_active_request_set(active, rq); GEM_BUG_ON(!ref->count); - return 0; +out: + i915_active_release(ref); + return err; } bool i915_active_acquire(struct i915_active *ref) @@ -223,19 +231,25 @@ int i915_request_await_active_request(struct i915_request *rq, int i915_request_await_active(struct i915_request *rq, struct i915_active *ref) { struct active_node *it, *n; - int ret; + int err = 0; - ret = i915_request_await_active_request(rq, &ref->last); - if (ret) - return ret; + /* await allocates and so we need to avoid hitting the shrinker */ + if (i915_active_acquire(ref)) + goto out; /* was idle */ + + err = i915_request_await_active_request(rq, &ref->last); + if (err) + goto out; rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { - ret = i915_request_await_active_request(rq, &it->base); - if (ret) - return ret; + err = i915_request_await_active_request(rq, &it->base); + if (err) + goto out; } - return 0; +out: + i915_active_release(ref); + return err; } #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 6630212f2faf3..9df65d386d11b 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -757,39 +757,6 @@ static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) return ret; } -#if !defined(CONFIG_VGA_CONSOLE) -static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) -{ - return 0; -} -#elif !defined(CONFIG_DUMMY_CONSOLE) -static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) -{ - return -ENODEV; -} -#else -static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) -{ - int ret = 0; - - DRM_INFO("Replacing VGA console driver\n"); - - console_lock(); - if (con_is_bound(&vga_con)) - ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1); - if (ret == 0) { - ret = do_unregister_con_driver(&vga_con); - - /* Ignore "already unregistered". */ - if (ret == -ENODEV) - ret = 0; - } - console_unlock(); - - return ret; -} -#endif - static void intel_init_dpio(struct drm_i915_private *dev_priv) { /* @@ -1420,7 +1387,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv) goto err_ggtt; } - ret = i915_kick_out_vgacon(dev_priv); + ret = vga_remove_vgacon(pdev); if (ret) { DRM_ERROR("failed to remove conflicting VGA console\n"); goto err_ggtt; diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 6728ea5c71d4c..30d516e975c64 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -1688,7 +1688,8 @@ __vma_matches(struct vm_area_struct *vma, struct file *filp, if (vma->vm_file != filp) return false; - return vma->vm_start == addr && (vma->vm_end - vma->vm_start) == size; + return vma->vm_start == addr && + (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size); } /** diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c index d01683167c774..8bc042551692c 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.c +++ b/drivers/gpu/drm/i915/i915_scheduler.c @@ -223,8 +223,14 @@ out: return &p->requests[idx]; } +struct sched_cache { + struct list_head *priolist; +}; + static struct intel_engine_cs * -sched_lock_engine(struct i915_sched_node *node, struct intel_engine_cs *locked) +sched_lock_engine(const struct i915_sched_node *node, + struct intel_engine_cs *locked, + struct sched_cache *cache) { struct intel_engine_cs *engine = node_to_request(node)->engine; @@ -232,6 +238,7 @@ sched_lock_engine(struct i915_sched_node *node, struct intel_engine_cs *locked) if (engine != locked) { spin_unlock(&locked->timeline.lock); + memset(cache, 0, sizeof(*cache)); spin_lock(&engine->timeline.lock); } @@ -253,11 +260,11 @@ static bool inflight(const struct i915_request *rq, static void __i915_schedule(struct i915_request *rq, const struct i915_sched_attr *attr) { - struct list_head *uninitialized_var(pl); - struct intel_engine_cs *engine, *last; + struct intel_engine_cs *engine; struct i915_dependency *dep, *p; struct i915_dependency stack; const int prio = attr->priority; + struct sched_cache cache; LIST_HEAD(dfs); /* Needed in order to use the temporary link inside i915_dependency */ @@ -328,7 +335,7 @@ static void __i915_schedule(struct i915_request *rq, __list_del_entry(&stack.dfs_link); } - last = NULL; + memset(&cache, 0, sizeof(cache)); engine = rq->engine; spin_lock_irq(&engine->timeline.lock); @@ -338,7 +345,7 @@ static void __i915_schedule(struct i915_request *rq, INIT_LIST_HEAD(&dep->dfs_link); - engine = sched_lock_engine(node, engine); + engine = sched_lock_engine(node, engine, &cache); lockdep_assert_held(&engine->timeline.lock); /* Recheck after acquiring the engine->timeline.lock */ @@ -347,11 +354,11 @@ static void __i915_schedule(struct i915_request *rq, node->attr.priority = prio; if (!list_empty(&node->link)) { - if (last != engine) { - pl = i915_sched_lookup_priolist(engine, prio); - last = engine; - } - list_move_tail(&node->link, pl); + if (!cache.priolist) + cache.priolist = + i915_sched_lookup_priolist(engine, + prio); + list_move_tail(&node->link, cache.priolist); } else { /* * If the request is not in the priolist queue because diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c index cacaa1d04d174..09ed90c0ba007 100644 --- a/drivers/gpu/drm/i915/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c @@ -106,16 +106,6 @@ bool intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine) GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); - clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); - - /* - * We may race with direct invocation of - * dma_fence_signal(), e.g. i915_request_retire(), - * in which case we can skip processing it ourselves. - */ - if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, - &rq->fence.flags)) - continue; /* * Queue for execution after dropping the signaling @@ -123,6 +113,14 @@ bool intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine) * more signalers to the same context or engine. */ i915_request_get(rq); + + /* + * We may race with direct invocation of + * dma_fence_signal(), e.g. i915_request_retire(), + * so we need to acquire our reference to the request + * before we cancel the breadcrumb. + */ + clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); list_add_tail(&rq->signal_link, &signal); } diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c index ca705546a0abe..14d580cdefd3e 100644 --- a/drivers/gpu/drm/i915/intel_ddi.c +++ b/drivers/gpu/drm/i915/intel_ddi.c @@ -3568,6 +3568,13 @@ static void intel_ddi_update_pipe(struct intel_encoder *encoder, { if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) intel_ddi_update_pipe_dp(encoder, crtc_state, conn_state); + + if (conn_state->content_protection == + DRM_MODE_CONTENT_PROTECTION_DESIRED) + intel_hdcp_enable(to_intel_connector(conn_state->connector)); + else if (conn_state->content_protection == + DRM_MODE_CONTENT_PROTECTION_UNDESIRED) + intel_hdcp_disable(to_intel_connector(conn_state->connector)); } static void intel_ddi_set_fia_lane_count(struct intel_encoder *encoder, @@ -3962,12 +3969,7 @@ static int modeset_pipe(struct drm_crtc *crtc, goto out; ret = drm_atomic_commit(state); - if (ret) - goto out; - - return 0; - - out: +out: drm_atomic_state_put(state); return ret; diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c index d00d0bb077842..7eb58a9d1319f 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_context.c @@ -710,47 +710,45 @@ __sseu_prepare(struct drm_i915_private *i915, unsigned int flags, struct i915_gem_context *ctx, struct intel_engine_cs *engine, - struct igt_spinner **spin_out) + struct igt_spinner **spin) { - int ret = 0; - - if (flags & (TEST_BUSY | TEST_RESET)) { - struct igt_spinner *spin; - struct i915_request *rq; + struct i915_request *rq; + int ret; - spin = kzalloc(sizeof(*spin), GFP_KERNEL); - if (!spin) { - ret = -ENOMEM; - goto out; - } + *spin = NULL; + if (!(flags & (TEST_BUSY | TEST_RESET))) + return 0; - ret = igt_spinner_init(spin, i915); - if (ret) - return ret; + *spin = kzalloc(sizeof(**spin), GFP_KERNEL); + if (!*spin) + return -ENOMEM; - rq = igt_spinner_create_request(spin, ctx, engine, MI_NOOP); - if (IS_ERR(rq)) { - ret = PTR_ERR(rq); - igt_spinner_fini(spin); - kfree(spin); - goto out; - } + ret = igt_spinner_init(*spin, i915); + if (ret) + goto err_free; - i915_request_add(rq); + rq = igt_spinner_create_request(*spin, ctx, engine, MI_NOOP); + if (IS_ERR(rq)) { + ret = PTR_ERR(rq); + goto err_fini; + } - if (!igt_wait_for_spinner(spin, rq)) { - pr_err("%s: Spinner failed to start!\n", name); - igt_spinner_end(spin); - igt_spinner_fini(spin); - kfree(spin); - ret = -ETIMEDOUT; - goto out; - } + i915_request_add(rq); - *spin_out = spin; + if (!igt_wait_for_spinner(*spin, rq)) { + pr_err("%s: Spinner failed to start!\n", name); + ret = -ETIMEDOUT; + goto err_end; } -out: + return 0; + +err_end: + igt_spinner_end(*spin); +err_fini: + igt_spinner_fini(*spin); +err_free: + kfree(fetch_and_zero(spin)); return ret; } @@ -897,22 +895,23 @@ __sseu_test(struct drm_i915_private *i915, ret = __sseu_prepare(i915, name, flags, ctx, engine, &spin); if (ret) - goto out; + goto out_context; ret = __i915_gem_context_reconfigure_sseu(ctx, engine, sseu); if (ret) - goto out; + goto out_spin; ret = __sseu_finish(i915, name, flags, ctx, kctx, engine, obj, hweight32(sseu.slice_mask), spin); -out: +out_spin: if (spin) { igt_spinner_end(spin); igt_spinner_fini(spin); kfree(spin); } +out_context: kernel_context_close(kctx); return ret; diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c index bb81e310eb6d3..578d867a81d59 100644 --- a/drivers/gpu/drm/qxl/qxl_drv.c +++ b/drivers/gpu/drm/qxl/qxl_drv.c @@ -79,6 +79,10 @@ qxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret) goto free_dev; + ret = drm_fb_helper_remove_conflicting_pci_framebuffers(pdev, 0, "qxl"); + if (ret) + goto disable_pci; + ret = qxl_device_init(qdev, &qxl_driver, pdev); if (ret) goto disable_pci; @@ -94,7 +98,6 @@ qxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret) goto modeset_cleanup; - drm_fb_helper_remove_conflicting_pci_framebuffers(pdev, 0, "qxl"); drm_fbdev_generic_setup(&qdev->ddev, 32); return 0; diff --git a/drivers/gpu/vga/vgaarb.c b/drivers/gpu/vga/vgaarb.c index dc8e039bfab57..f2f3ef8af2710 100644 --- a/drivers/gpu/vga/vgaarb.c +++ b/drivers/gpu/vga/vgaarb.c @@ -48,6 +48,8 @@ #include <linux/miscdevice.h> #include <linux/slab.h> #include <linux/screen_info.h> +#include <linux/vt.h> +#include <linux/console.h> #include <linux/uaccess.h> @@ -168,6 +170,53 @@ void vga_set_default_device(struct pci_dev *pdev) vga_default = pci_dev_get(pdev); } +/** + * vga_remove_vgacon - deactivete vga console + * + * Unbind and unregister vgacon in case pdev is the default vga + * device. Can be called by gpu drivers on initialization to make + * sure vga register access done by vgacon will not disturb the + * device. + * + * @pdev: pci device. + */ +#if !defined(CONFIG_VGA_CONSOLE) +int vga_remove_vgacon(struct pci_dev *pdev) +{ + return 0; +} +#elif !defined(CONFIG_DUMMY_CONSOLE) +int vga_remove_vgacon(struct pci_dev *pdev) +{ + return -ENODEV; +} +#else +int vga_remove_vgacon(struct pci_dev *pdev) +{ + int ret = 0; + + if (pdev != vga_default) + return 0; + vgaarb_info(&pdev->dev, "deactivate vga console\n"); + + console_lock(); + if (con_is_bound(&vga_con)) + ret = do_take_over_console(&dummy_con, 0, + MAX_NR_CONSOLES - 1, 1); + if (ret == 0) { + ret = do_unregister_con_driver(&vga_con); + + /* Ignore "already unregistered". */ + if (ret == -ENODEV) + ret = 0; + } + console_unlock(); + + return ret; +} +#endif +EXPORT_SYMBOL(vga_remove_vgacon); + static inline void vga_irq_set_state(struct vga_device *vgadev, bool state) { if (vgadev->irq_set_state) diff --git a/drivers/hwtracing/coresight/coresight-etm3x.c b/drivers/hwtracing/coresight/coresight-etm3x.c index 9a63e87ea5f3f..be302ec5f66bd 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x.c +++ b/drivers/hwtracing/coresight/coresight-etm3x.c @@ -871,7 +871,7 @@ static int etm_probe(struct amba_device *adev, const struct amba_id *id) } pm_runtime_put(&adev->dev); - dev_info(dev, "%s initialized\n", (char *)id->data); + dev_info(dev, "%s initialized\n", (char *)coresight_get_uci_data(id)); if (boot_enable) { coresight_enable(drvdata->csdev); drvdata->boot_enable = true; @@ -915,36 +915,18 @@ static const struct dev_pm_ops etm_dev_pm_ops = { }; static const struct amba_id etm_ids[] = { - { /* ETM 3.3 */ - .id = 0x000bb921, - .mask = 0x000fffff, - .data = "ETM 3.3", - }, - { /* ETM 3.5 - Cortex-A5 */ - .id = 0x000bb955, - .mask = 0x000fffff, - .data = "ETM 3.5", - }, - { /* ETM 3.5 */ - .id = 0x000bb956, - .mask = 0x000fffff, - .data = "ETM 3.5", - }, - { /* PTM 1.0 */ - .id = 0x000bb950, - .mask = 0x000fffff, - .data = "PTM 1.0", - }, - { /* PTM 1.1 */ - .id = 0x000bb95f, - .mask = 0x000fffff, - .data = "PTM 1.1", - }, - { /* PTM 1.1 Qualcomm */ - .id = 0x000b006f, - .mask = 0x000fffff, - .data = "PTM 1.1", - }, + /* ETM 3.3 */ + CS_AMBA_ID_DATA(0x000bb921, "ETM 3.3"), + /* ETM 3.5 - Cortex-A5 */ + CS_AMBA_ID_DATA(0x000bb955, "ETM 3.5"), + /* ETM 3.5 */ + CS_AMBA_ID_DATA(0x000bb956, "ETM 3.5"), + /* PTM 1.0 */ + CS_AMBA_ID_DATA(0x000bb950, "PTM 1.0"), + /* PTM 1.1 */ + CS_AMBA_ID_DATA(0x000bb95f, "PTM 1.1"), + /* PTM 1.1 Qualcomm */ + CS_AMBA_ID_DATA(0x000b006f, "PTM 1.1"), { 0, 0}, }; diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c index fe76b176974a6..08ce37c9475da 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.c +++ b/drivers/hwtracing/coresight/coresight-etm4x.c @@ -1068,18 +1068,21 @@ err_arch_supported: return ret; } -#define ETM4x_AMBA_ID(pid) \ - { \ - .id = pid, \ - .mask = 0x000fffff, \ +static struct amba_cs_uci_id uci_id_etm4[] = { + { + /* ETMv4 UCI data */ + .devarch = 0x47704a13, + .devarch_mask = 0xfff0ffff, + .devtype = 0x00000013, } +}; static const struct amba_id etm4_ids[] = { - ETM4x_AMBA_ID(0x000bb95d), /* Cortex-A53 */ - ETM4x_AMBA_ID(0x000bb95e), /* Cortex-A57 */ - ETM4x_AMBA_ID(0x000bb95a), /* Cortex-A72 */ - ETM4x_AMBA_ID(0x000bb959), /* Cortex-A73 */ - ETM4x_AMBA_ID(0x000bb9da), /* Cortex-A35 */ + CS_AMBA_ID(0x000bb95d), /* Cortex-A53 */ + CS_AMBA_ID(0x000bb95e), /* Cortex-A57 */ + CS_AMBA_ID(0x000bb95a), /* Cortex-A72 */ + CS_AMBA_ID(0x000bb959), /* Cortex-A73 */ + CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4), /* Cortex-A35 */ {}, }; diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index b936c6d7e13f3..e0684d06e9ee9 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -6,6 +6,7 @@ #ifndef _CORESIGHT_PRIV_H #define _CORESIGHT_PRIV_H +#include <linux/amba/bus.h> #include <linux/bitops.h> #include <linux/io.h> #include <linux/coresight.h> @@ -160,4 +161,43 @@ static inline int etm_readl_cp14(u32 off, unsigned int *val) { return 0; } static inline int etm_writel_cp14(u32 off, u32 val) { return 0; } #endif +/* + * Macros and inline functions to handle CoreSight UCI data and driver + * private data in AMBA ID table entries, and extract data values. + */ + +/* coresight AMBA ID, no UCI, no driver data: id table entry */ +#define CS_AMBA_ID(pid) \ + { \ + .id = pid, \ + .mask = 0x000fffff, \ + } + +/* coresight AMBA ID, UCI with driver data only: id table entry. */ +#define CS_AMBA_ID_DATA(pid, dval) \ + { \ + .id = pid, \ + .mask = 0x000fffff, \ + .data = (void *)&(struct amba_cs_uci_id) \ + { \ + .data = (void *)dval, \ + } \ + } + +/* coresight AMBA ID, full UCI structure: id table entry. */ +#define CS_AMBA_UCI_ID(pid, uci_ptr) \ + { \ + .id = pid, \ + .mask = 0x000fffff, \ + .data = uci_ptr \ + } + +/* extract the data value from a UCI structure given amba_id pointer. */ +static inline void *coresight_get_uci_data(const struct amba_id *id) +{ + if (id->data) + return ((struct amba_cs_uci_id *)(id->data))->data; + return 0; +} + #endif diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index f07825df5c7a9..9f8a844ed7aa3 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -870,7 +870,7 @@ static int stm_probe(struct amba_device *adev, const struct amba_id *id) pm_runtime_put(&adev->dev); - dev_info(dev, "%s initialized\n", (char *)id->data); + dev_info(dev, "%s initialized\n", (char *)coresight_get_uci_data(id)); return 0; stm_unregister: @@ -905,16 +905,8 @@ static const struct dev_pm_ops stm_dev_pm_ops = { }; static const struct amba_id stm_ids[] = { - { - .id = 0x000bb962, - .mask = 0x000fffff, - .data = "STM32", - }, - { - .id = 0x000bb963, - .mask = 0x000fffff, - .data = "STM500", - }, + CS_AMBA_ID_DATA(0x000bb962, "STM32"), + CS_AMBA_ID_DATA(0x000bb963, "STM500"), { 0, 0}, }; diff --git a/drivers/hwtracing/coresight/coresight-tmc.c b/drivers/hwtracing/coresight/coresight-tmc.c index ea249f0bcd73c..2a02da3d630f4 100644 --- a/drivers/hwtracing/coresight/coresight-tmc.c +++ b/drivers/hwtracing/coresight/coresight-tmc.c @@ -443,7 +443,8 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id) desc.type = CORESIGHT_DEV_TYPE_SINK; desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER; desc.ops = &tmc_etr_cs_ops; - ret = tmc_etr_setup_caps(drvdata, devid, id->data); + ret = tmc_etr_setup_caps(drvdata, devid, + coresight_get_uci_data(id)); if (ret) goto out; break; @@ -475,26 +476,13 @@ out: } static const struct amba_id tmc_ids[] = { - { - .id = 0x000bb961, - .mask = 0x000fffff, - }, - { - /* Coresight SoC 600 TMC-ETR/ETS */ - .id = 0x000bb9e8, - .mask = 0x000fffff, - .data = (void *)(unsigned long)CORESIGHT_SOC_600_ETR_CAPS, - }, - { - /* Coresight SoC 600 TMC-ETB */ - .id = 0x000bb9e9, - .mask = 0x000fffff, - }, - { - /* Coresight SoC 600 TMC-ETF */ - .id = 0x000bb9ea, - .mask = 0x000fffff, - }, + CS_AMBA_ID(0x000bb961), + /* Coresight SoC 600 TMC-ETR/ETS */ + CS_AMBA_ID_DATA(0x000bb9e8, (unsigned long)CORESIGHT_SOC_600_ETR_CAPS), + /* Coresight SoC 600 TMC-ETB */ + CS_AMBA_ID(0x000bb9e9), + /* Coresight SoC 600 TMC-ETF */ + CS_AMBA_ID(0x000bb9ea), { 0, 0}, }; diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index ead5e7de3e4d1..416f89b8f8812 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -86,7 +86,6 @@ static int dw_i2c_acpi_configure(struct platform_device *pdev) struct i2c_timings *t = &dev->timings; u32 ss_ht = 0, fp_ht = 0, hs_ht = 0, fs_ht = 0; - dev->adapter.nr = -1; dev->tx_fifo_depth = 32; dev->rx_fifo_depth = 32; @@ -219,7 +218,7 @@ static void i2c_dw_configure_slave(struct dw_i2c_dev *dev) dev->mode = DW_IC_SLAVE; } -static void dw_i2c_set_fifo_size(struct dw_i2c_dev *dev, int id) +static void dw_i2c_set_fifo_size(struct dw_i2c_dev *dev) { u32 param, tx_fifo_depth, rx_fifo_depth; @@ -233,7 +232,6 @@ static void dw_i2c_set_fifo_size(struct dw_i2c_dev *dev, int id) if (!dev->tx_fifo_depth) { dev->tx_fifo_depth = tx_fifo_depth; dev->rx_fifo_depth = rx_fifo_depth; - dev->adapter.nr = id; } else if (tx_fifo_depth >= 2) { dev->tx_fifo_depth = min_t(u32, dev->tx_fifo_depth, tx_fifo_depth); @@ -358,13 +356,14 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) div_u64(clk_khz * t->sda_hold_ns + 500000, 1000000); } - dw_i2c_set_fifo_size(dev, pdev->id); + dw_i2c_set_fifo_size(dev); adap = &dev->adapter; adap->owner = THIS_MODULE; adap->class = I2C_CLASS_DEPRECATED; ACPI_COMPANION_SET(&adap->dev, ACPI_COMPANION(&pdev->dev)); adap->dev.of_node = pdev->dev.of_node; + adap->nr = -1; dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_SMART_PREPARE | diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c index 660de1ee68ed4..684d651612b30 100644 --- a/drivers/i2c/busses/i2c-mt65xx.c +++ b/drivers/i2c/busses/i2c-mt65xx.c @@ -503,7 +503,7 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs, writel(I2C_DMA_INT_FLAG_NONE, i2c->pdmabase + OFFSET_INT_FLAG); writel(I2C_DMA_CON_RX, i2c->pdmabase + OFFSET_CON); - dma_rd_buf = i2c_get_dma_safe_msg_buf(msgs, 0); + dma_rd_buf = i2c_get_dma_safe_msg_buf(msgs, 1); if (!dma_rd_buf) return -ENOMEM; @@ -526,7 +526,7 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs, writel(I2C_DMA_INT_FLAG_NONE, i2c->pdmabase + OFFSET_INT_FLAG); writel(I2C_DMA_CON_TX, i2c->pdmabase + OFFSET_CON); - dma_wr_buf = i2c_get_dma_safe_msg_buf(msgs, 0); + dma_wr_buf = i2c_get_dma_safe_msg_buf(msgs, 1); if (!dma_wr_buf) return -ENOMEM; @@ -549,7 +549,7 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs, writel(I2C_DMA_CLR_FLAG, i2c->pdmabase + OFFSET_INT_FLAG); writel(I2C_DMA_CLR_FLAG, i2c->pdmabase + OFFSET_CON); - dma_wr_buf = i2c_get_dma_safe_msg_buf(msgs, 0); + dma_wr_buf = i2c_get_dma_safe_msg_buf(msgs, 1); if (!dma_wr_buf) return -ENOMEM; @@ -561,7 +561,7 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs, return -ENOMEM; } - dma_rd_buf = i2c_get_dma_safe_msg_buf((msgs + 1), 0); + dma_rd_buf = i2c_get_dma_safe_msg_buf((msgs + 1), 1); if (!dma_rd_buf) { dma_unmap_single(i2c->dev, wpaddr, msgs->len, DMA_TO_DEVICE); diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c index dd52a068b1406..a7578f6da9796 100644 --- a/drivers/i2c/busses/i2c-rcar.c +++ b/drivers/i2c/busses/i2c-rcar.c @@ -363,9 +363,6 @@ static void rcar_i2c_dma_unmap(struct rcar_i2c_priv *priv) struct dma_chan *chan = priv->dma_direction == DMA_FROM_DEVICE ? priv->dma_rx : priv->dma_tx; - /* Disable DMA Master Received/Transmitted */ - rcar_i2c_write(priv, ICDMAER, 0); - dma_unmap_single(chan->device->dev, sg_dma_address(&priv->sg), sg_dma_len(&priv->sg), priv->dma_direction); @@ -375,6 +372,9 @@ static void rcar_i2c_dma_unmap(struct rcar_i2c_priv *priv) priv->flags |= ID_P_NO_RXDMA; priv->dma_direction = DMA_NONE; + + /* Disable DMA Master Received/Transmitted, must be last! */ + rcar_i2c_write(priv, ICDMAER, 0); } static void rcar_i2c_cleanup_dma(struct rcar_i2c_priv *priv) @@ -611,6 +611,15 @@ static bool rcar_i2c_slave_irq(struct rcar_i2c_priv *priv) return true; } +/* + * This driver has a lock-free design because there are IP cores (at least + * R-Car Gen2) which have an inherent race condition in their hardware design. + * There, we need to clear RCAR_BUS_MASK_DATA bits as soon as possible after + * the interrupt was generated, otherwise an unwanted repeated message gets + * generated. It turned out that taking a spinlock at the beginning of the ISR + * was already causing repeated messages. Thus, this driver was converted to + * the now lockless behaviour. Please keep this in mind when hacking the driver. + */ static irqreturn_t rcar_i2c_irq(int irq, void *ptr) { struct rcar_i2c_priv *priv = ptr; diff --git a/drivers/i2c/busses/i2c-sis630.c b/drivers/i2c/busses/i2c-sis630.c index 1e6805b5cef23..a57aa4fe51a45 100644 --- a/drivers/i2c/busses/i2c-sis630.c +++ b/drivers/i2c/busses/i2c-sis630.c @@ -478,7 +478,7 @@ static int sis630_setup(struct pci_dev *sis630_dev) if (!request_region(smbus_base + SMB_STS, SIS630_SMB_IOREGION, sis630_driver.name)) { dev_err(&sis630_dev->dev, - "I/O Region 0x%04hx-0x%04hx for SMBus already in use.\n", + "I/O Region 0x%04x-0x%04x for SMBus already in use.\n", smbus_base + SMB_STS, smbus_base + SMB_STS + SIS630_SMB_IOREGION - 1); retval = -EBUSY; @@ -528,7 +528,7 @@ static int sis630_probe(struct pci_dev *dev, const struct pci_device_id *id) sis630_adapter.dev.parent = &dev->dev; snprintf(sis630_adapter.name, sizeof(sis630_adapter.name), - "SMBus SIS630 adapter at %04hx", smbus_base + SMB_STS); + "SMBus SIS630 adapter at %04x", smbus_base + SMB_STS); return i2c_add_adapter(&sis630_adapter); } diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c index 13e1213561d4b..4284fc991cfd4 100644 --- a/drivers/i2c/busses/i2c-stm32f7.c +++ b/drivers/i2c/busses/i2c-stm32f7.c @@ -432,7 +432,7 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev, STM32F7_I2C_ANALOG_FILTER_DELAY_MAX : 0); dnf_delay = setup->dnf * i2cclk; - sdadel_min = setup->fall_time - i2c_specs[setup->speed].hddat_min - + sdadel_min = i2c_specs[setup->speed].hddat_min + setup->fall_time - af_delay_min - (setup->dnf + 3) * i2cclk; sdadel_max = i2c_specs[setup->speed].vddat_max - setup->rise_time - diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index cb6c5cb0df0b3..38af18645133c 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -2258,7 +2258,8 @@ EXPORT_SYMBOL(i2c_put_adapter); /** * i2c_get_dma_safe_msg_buf() - get a DMA safe buffer for the given i2c_msg * @msg: the message to be checked - * @threshold: the minimum number of bytes for which using DMA makes sense + * @threshold: the minimum number of bytes for which using DMA makes sense. + * Should at least be 1. * * Return: NULL if a DMA safe buffer was not obtained. Use msg->buf with PIO. * Or a valid pointer to be used with DMA. After use, release it by @@ -2268,7 +2269,11 @@ EXPORT_SYMBOL(i2c_put_adapter); */ u8 *i2c_get_dma_safe_msg_buf(struct i2c_msg *msg, unsigned int threshold) { - if (msg->len < threshold) + /* also skip 0-length msgs for bogus thresholds of 0 */ + if (!threshold) + pr_debug("DMA buffer for addr=0x%02x with length 0 is bogus\n", + msg->addr); + if (msg->len < threshold || msg->len == 0) return NULL; if (msg->flags & I2C_M_DMA_SAFE) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 6b0760dafb3e9..b319e51c379bd 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -140,10 +140,14 @@ static struct lock_class_key reserved_rbtree_key; static inline int match_hid_uid(struct device *dev, struct acpihid_map_entry *entry) { + struct acpi_device *adev = ACPI_COMPANION(dev); const char *hid, *uid; - hid = acpi_device_hid(ACPI_COMPANION(dev)); - uid = acpi_device_uid(ACPI_COMPANION(dev)); + if (!adev) + return -ENODEV; + + hid = acpi_device_hid(adev); + uid = acpi_device_uid(adev); if (!hid || !(*hid)) return -ENODEV; diff --git a/drivers/isdn/hardware/mISDN/hfcpci.c b/drivers/isdn/hardware/mISDN/hfcpci.c index ebb3fa2e1d00f..362aa5450a5ec 100644 --- a/drivers/isdn/hardware/mISDN/hfcpci.c +++ b/drivers/isdn/hardware/mISDN/hfcpci.c @@ -2032,10 +2032,19 @@ setup_hw(struct hfc_pci *hc) hc->hw.fifos = buffer; pci_write_config_dword(hc->pdev, 0x80, hc->hw.dmahandle); hc->hw.pci_io = ioremap((ulong) hc->hw.pci_io, 256); + if (unlikely(!hc->hw.pci_io)) { + printk(KERN_WARNING + "HFC-PCI: Error in ioremap for PCI!\n"); + pci_free_consistent(hc->pdev, 0x8000, hc->hw.fifos, + hc->hw.dmahandle); + return 1; + } + printk(KERN_INFO "HFC-PCI: defined at mem %#lx fifo %#lx(%#lx) IRQ %d HZ %d\n", (u_long) hc->hw.pci_io, (u_long) hc->hw.fifos, (u_long) hc->hw.dmahandle, hc->irq, HZ); + /* enable memory mapped ports, disable busmaster */ pci_write_config_word(hc->pdev, PCI_COMMAND, PCI_ENA_MEMIO); hc->hw.int_m2 = 0; diff --git a/drivers/lightnvm/pblk-rl.c b/drivers/lightnvm/pblk-rl.c index b014957dde0ba..a5f8bc2defbcd 100644 --- a/drivers/lightnvm/pblk-rl.c +++ b/drivers/lightnvm/pblk-rl.c @@ -233,10 +233,15 @@ void pblk_rl_init(struct pblk_rl *rl, int budget, int threshold) /* To start with, all buffer is available to user I/O writers */ rl->rb_budget = budget; rl->rb_user_max = budget; - rl->rb_max_io = threshold ? (budget - threshold) : (budget - 1); rl->rb_gc_max = 0; rl->rb_state = PBLK_RL_HIGH; + /* Maximize I/O size and ansure that back threshold is respected */ + if (threshold) + rl->rb_max_io = budget - pblk->min_write_pgs_data - threshold; + else + rl->rb_max_io = budget - pblk->min_write_pgs_data - 1; + atomic_set(&rl->rb_user_cnt, 0); atomic_set(&rl->rb_gc_cnt, 0); atomic_set(&rl->rb_space, -1); diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index abb5d382f64d1..3b6880dd648d2 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -3939,6 +3939,8 @@ static int raid10_run(struct mddev *mddev) set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); mddev->sync_thread = md_register_thread(md_do_sync, mddev, "reshape"); + if (!mddev->sync_thread) + goto out_free_conf; } return 0; @@ -4670,7 +4672,6 @@ read_more: atomic_inc(&r10_bio->remaining); read_bio->bi_next = NULL; generic_make_request(read_bio); - sector_nr += nr_sectors; sectors_done += nr_sectors; if (sector_nr <= last) goto read_more; diff --git a/drivers/md/raid5-log.h b/drivers/md/raid5-log.h index bfb8114070614..43c714a8798c5 100644 --- a/drivers/md/raid5-log.h +++ b/drivers/md/raid5-log.h @@ -45,6 +45,7 @@ extern void ppl_stripe_write_finished(struct stripe_head *sh); extern int ppl_modify_log(struct r5conf *conf, struct md_rdev *rdev, bool add); extern void ppl_quiesce(struct r5conf *conf, int quiesce); extern int ppl_handle_flush_request(struct r5l_log *log, struct bio *bio); +extern struct md_sysfs_entry ppl_write_hint; static inline bool raid5_has_log(struct r5conf *conf) { diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c index 0b096ddc9c1e6..17e9e7d510978 100644 --- a/drivers/md/raid5-ppl.c +++ b/drivers/md/raid5-ppl.c @@ -20,6 +20,7 @@ #include <linux/raid/md_p.h> #include "md.h" #include "raid5.h" +#include "raid5-log.h" /* * PPL consists of a 4KB header (struct ppl_header) and at least 128KB for @@ -115,6 +116,8 @@ struct ppl_conf { /* stripes to retry if failed to allocate io_unit */ struct list_head no_mem_stripes; spinlock_t no_mem_stripes_lock; + + unsigned short write_hint; }; struct ppl_log { @@ -474,6 +477,7 @@ static void ppl_submit_iounit(struct ppl_io_unit *io) bio_set_dev(bio, log->rdev->bdev); bio->bi_iter.bi_sector = log->next_io_sector; bio_add_page(bio, io->header_page, PAGE_SIZE, 0); + bio->bi_write_hint = ppl_conf->write_hint; pr_debug("%s: log->current_io_sector: %llu\n", __func__, (unsigned long long)log->next_io_sector); @@ -503,6 +507,7 @@ static void ppl_submit_iounit(struct ppl_io_unit *io) bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, &ppl_conf->bs); bio->bi_opf = prev->bi_opf; + bio->bi_write_hint = prev->bi_write_hint; bio_copy_dev(bio, prev); bio->bi_iter.bi_sector = bio_end_sector(prev); bio_add_page(bio, sh->ppl_page, PAGE_SIZE, 0); @@ -1407,6 +1412,7 @@ int ppl_init_log(struct r5conf *conf) atomic64_set(&ppl_conf->seq, 0); INIT_LIST_HEAD(&ppl_conf->no_mem_stripes); spin_lock_init(&ppl_conf->no_mem_stripes_lock); + ppl_conf->write_hint = RWF_WRITE_LIFE_NOT_SET; if (!mddev->external) { ppl_conf->signature = ~crc32c_le(~0, mddev->uuid, sizeof(mddev->uuid)); @@ -1501,3 +1507,60 @@ int ppl_modify_log(struct r5conf *conf, struct md_rdev *rdev, bool add) return ret; } + +static ssize_t +ppl_write_hint_show(struct mddev *mddev, char *buf) +{ + size_t ret = 0; + struct r5conf *conf; + struct ppl_conf *ppl_conf = NULL; + + spin_lock(&mddev->lock); + conf = mddev->private; + if (conf && raid5_has_ppl(conf)) + ppl_conf = conf->log_private; + ret = sprintf(buf, "%d\n", ppl_conf ? ppl_conf->write_hint : 0); + spin_unlock(&mddev->lock); + + return ret; +} + +static ssize_t +ppl_write_hint_store(struct mddev *mddev, const char *page, size_t len) +{ + struct r5conf *conf; + struct ppl_conf *ppl_conf; + int err = 0; + unsigned short new; + + if (len >= PAGE_SIZE) + return -EINVAL; + if (kstrtou16(page, 10, &new)) + return -EINVAL; + + err = mddev_lock(mddev); + if (err) + return err; + + conf = mddev->private; + if (!conf) { + err = -ENODEV; + } else if (raid5_has_ppl(conf)) { + ppl_conf = conf->log_private; + if (!ppl_conf) + err = -EINVAL; + else + ppl_conf->write_hint = new; + } else { + err = -EINVAL; + } + + mddev_unlock(mddev); + + return err ?: len; +} + +struct md_sysfs_entry +ppl_write_hint = __ATTR(ppl_write_hint, S_IRUGO | S_IWUSR, + ppl_write_hint_show, + ppl_write_hint_store); diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 77ffd09be486d..c033bfcb209e4 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -6650,6 +6650,7 @@ static struct attribute *raid5_attrs[] = { &raid5_skip_copy.attr, &raid5_rmw_level.attr, &r5c_journal_mode.attr, + &ppl_write_hint.attr, NULL, }; static struct attribute_group raid5_attrs_group = { @@ -7393,6 +7394,8 @@ static int raid5_run(struct mddev *mddev) set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); mddev->sync_thread = md_register_thread(md_do_sync, mddev, "reshape"); + if (!mddev->sync_thread) + goto abort; } /* Ok, everything is just fine now */ diff --git a/drivers/net/ethernet/8390/pcnet_cs.c b/drivers/net/ethernet/8390/pcnet_cs.c index 61e43802b9a59..645efac6310dd 100644 --- a/drivers/net/ethernet/8390/pcnet_cs.c +++ b/drivers/net/ethernet/8390/pcnet_cs.c @@ -289,6 +289,11 @@ static struct hw_info *get_hwinfo(struct pcmcia_device *link) virt = ioremap(link->resource[2]->start, resource_size(link->resource[2])); + if (unlikely(!virt)) { + pcmcia_release_window(link, link->resource[2]); + return NULL; + } + for (i = 0; i < NR_INFO; i++) { pcmcia_map_mem_page(link, link->resource[2], hw_info[i].offset & ~(resource_size(link->resource[2])-1)); @@ -1423,6 +1428,11 @@ static int setup_shmem_window(struct pcmcia_device *link, int start_pg, /* Try scribbling on the buffer */ info->base = ioremap(link->resource[3]->start, resource_size(link->resource[3])); + if (unlikely(!info->base)) { + ret = -ENOMEM; + goto failed; + } + for (i = 0; i < (TX_PAGES<<8); i += 2) __raw_writew((i>>1), info->base+offset+i); udelay(100); diff --git a/drivers/net/ethernet/cavium/liquidio/lio_core.c b/drivers/net/ethernet/cavium/liquidio/lio_core.c index e21bf37246113..1c50c10b5a16b 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_core.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_core.c @@ -1211,6 +1211,11 @@ int liquidio_change_mtu(struct net_device *netdev, int new_mtu) sc = (struct octeon_soft_command *) octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE, 16, 0); + if (!sc) { + netif_info(lio, rx_err, lio->netdev, + "Failed to allocate soft command\n"); + return -ENOMEM; + } ncmd = (union octnet_cmd *)sc->virtdptr; @@ -1684,6 +1689,11 @@ int liquidio_set_fec(struct lio *lio, int on_off) sc = octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE, sizeof(struct oct_nic_seapi_resp), 0); + if (!sc) { + dev_err(&oct->pci_dev->dev, + "Failed to allocate soft command\n"); + return -ENOMEM; + } ncmd = sc->virtdptr; resp = sc->virtrptr; diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c index 9b7819fdc9ded..fb6f813cff65d 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c @@ -1192,6 +1192,11 @@ static void send_rx_ctrl_cmd(struct lio *lio, int start_stop) sc = (struct octeon_soft_command *) octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE, 16, 0); + if (!sc) { + netif_info(lio, rx_err, lio->netdev, + "Failed to allocate octeon_soft_command\n"); + return; + } ncmd = (union octnet_cmd *)sc->virtdptr; diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c index 503cfadff4ace..aa2be48071913 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c @@ -2234,6 +2234,12 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) nic->nicvf_rx_mode_wq = alloc_ordered_workqueue("nicvf_rx_mode_wq_VF%d", WQ_MEM_RECLAIM, nic->vf_id); + if (!nic->nicvf_rx_mode_wq) { + err = -ENOMEM; + dev_err(dev, "Failed to allocate work queue\n"); + goto err_unregister_interrupts; + } + INIT_WORK(&nic->rx_mode_work.work, nicvf_set_rx_mode_task); spin_lock_init(&nic->rx_mode_wq_lock); mutex_init(&nic->rx_mode_mtx); diff --git a/drivers/net/ethernet/fujitsu/fmvj18x_cs.c b/drivers/net/ethernet/fujitsu/fmvj18x_cs.c index a69cd19a55ae2..1eca0fdb99334 100644 --- a/drivers/net/ethernet/fujitsu/fmvj18x_cs.c +++ b/drivers/net/ethernet/fujitsu/fmvj18x_cs.c @@ -547,6 +547,11 @@ static int fmvj18x_get_hwinfo(struct pcmcia_device *link, u_char *node_id) return -1; base = ioremap(link->resource[2]->start, resource_size(link->resource[2])); + if (!base) { + pcmcia_release_window(link, link->resource[2]); + return -ENOMEM; + } + pcmcia_map_mem_page(link, link->resource[2], 0); /* diff --git a/drivers/net/ethernet/mellanox/mlx4/cmd.c b/drivers/net/ethernet/mellanox/mlx4/cmd.c index c19e74e6ac94d..a5d5d6fc1da00 100644 --- a/drivers/net/ethernet/mellanox/mlx4/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx4/cmd.c @@ -2645,6 +2645,8 @@ int mlx4_cmd_use_events(struct mlx4_dev *dev) if (!priv->cmd.context) return -ENOMEM; + if (mlx4_is_mfunc(dev)) + mutex_lock(&priv->cmd.slave_cmd_mutex); down_write(&priv->cmd.switch_sem); for (i = 0; i < priv->cmd.max_cmds; ++i) { priv->cmd.context[i].token = i; @@ -2670,6 +2672,8 @@ int mlx4_cmd_use_events(struct mlx4_dev *dev) down(&priv->cmd.poll_sem); priv->cmd.use_events = 1; up_write(&priv->cmd.switch_sem); + if (mlx4_is_mfunc(dev)) + mutex_unlock(&priv->cmd.slave_cmd_mutex); return err; } @@ -2682,6 +2686,8 @@ void mlx4_cmd_use_polling(struct mlx4_dev *dev) struct mlx4_priv *priv = mlx4_priv(dev); int i; + if (mlx4_is_mfunc(dev)) + mutex_lock(&priv->cmd.slave_cmd_mutex); down_write(&priv->cmd.switch_sem); priv->cmd.use_events = 0; @@ -2689,9 +2695,12 @@ void mlx4_cmd_use_polling(struct mlx4_dev *dev) down(&priv->cmd.event_sem); kfree(priv->cmd.context); + priv->cmd.context = NULL; up(&priv->cmd.poll_sem); up_write(&priv->cmd.switch_sem); + if (mlx4_is_mfunc(dev)) + mutex_unlock(&priv->cmd.slave_cmd_mutex); } struct mlx4_cmd_mailbox *mlx4_alloc_cmd_mailbox(struct mlx4_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c index eb13d36181624..4356f3a580027 100644 --- a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c +++ b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c @@ -2719,13 +2719,13 @@ static int qp_get_mtt_size(struct mlx4_qp_context *qpc) int total_pages; int total_mem; int page_offset = (be32_to_cpu(qpc->params2) >> 6) & 0x3f; + int tot; sq_size = 1 << (log_sq_size + log_sq_sride + 4); rq_size = (srq|rss|xrc) ? 0 : (1 << (log_rq_size + log_rq_stride + 4)); total_mem = sq_size + rq_size; - total_pages = - roundup_pow_of_two((total_mem + (page_offset << 6)) >> - page_shift); + tot = (total_mem + (page_offset << 6)) >> page_shift; + total_pages = !tot ? 1 : roundup_pow_of_two(tot); return total_pages; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 0804b478ad192..a0987cc5fe4a1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -424,6 +424,9 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { priv->channels.params = new_channels.params; + if (!netif_is_rxfh_configured(priv->netdev)) + mlx5e_build_default_indir_rqt(priv->rss_params.indirection_rqt, + MLX5E_INDIR_RQT_SIZE, count); goto out; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index a1a3e27749890..a66b6ed80b302 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -1129,16 +1129,17 @@ static int mlx5e_rep_get_phys_port_name(struct net_device *dev, struct mlx5e_priv *priv = netdev_priv(dev); struct mlx5e_rep_priv *rpriv = priv->ppriv; struct mlx5_eswitch_rep *rep = rpriv->rep; - int ret, pf_num; + unsigned int fn; + int ret; - ret = mlx5_lag_get_pf_num(priv->mdev, &pf_num); - if (ret) - return ret; + fn = PCI_FUNC(priv->mdev->pdev->devfn); + if (fn >= MLX5_MAX_PORTS) + return -EOPNOTSUPP; if (rep->vport == MLX5_VPORT_UPLINK) - ret = snprintf(buf, len, "p%d", pf_num); + ret = snprintf(buf, len, "p%d", fn); else - ret = snprintf(buf, len, "pf%dvf%d", pf_num, rep->vport - 1); + ret = snprintf(buf, len, "pf%dvf%d", fn, rep->vport - 1); if (ret >= len) return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index be396e5e4e39c..3dde5c7e0739a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -1295,8 +1295,14 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq, skb->protocol = *((__be16 *)(skb->data)); - skb->ip_summed = CHECKSUM_COMPLETE; - skb->csum = csum_unfold((__force __sum16)cqe->check_sum); + if (netdev->features & NETIF_F_RXCSUM) { + skb->ip_summed = CHECKSUM_COMPLETE; + skb->csum = csum_unfold((__force __sum16)cqe->check_sum); + stats->csum_complete++; + } else { + skb->ip_summed = CHECKSUM_NONE; + stats->csum_none++; + } if (unlikely(mlx5e_rx_hw_stamp(tstamp))) skb_hwtstamps(skb)->hwtstamp = @@ -1315,7 +1321,6 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq, skb->dev = netdev; - stats->csum_complete++; stats->packets++; stats->bytes += cqe_bcnt; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index d0b28251abf2b..ecd2c747f7260 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1931,7 +1931,7 @@ int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw, u64 node_guid; int err = 0; - if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) + if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager)) return -EPERM; if (!LEGAL_VPORT(esw, vport) || is_multicast_ether_addr(mac)) return -EINVAL; @@ -2005,7 +2005,7 @@ int mlx5_eswitch_get_vport_config(struct mlx5_eswitch *esw, { struct mlx5_vport *evport; - if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) + if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager)) return -EPERM; if (!LEGAL_VPORT(esw, vport)) return -EINVAL; @@ -2297,19 +2297,24 @@ static int normalize_vports_min_rate(struct mlx5_eswitch *esw, u32 divider) int mlx5_eswitch_set_vport_rate(struct mlx5_eswitch *esw, int vport, u32 max_rate, u32 min_rate) { - u32 fw_max_bw_share = MLX5_CAP_QOS(esw->dev, max_tsar_bw_share); - bool min_rate_supported = MLX5_CAP_QOS(esw->dev, esw_bw_share) && - fw_max_bw_share >= MLX5_MIN_BW_SHARE; - bool max_rate_supported = MLX5_CAP_QOS(esw->dev, esw_rate_limit); struct mlx5_vport *evport; + u32 fw_max_bw_share; u32 previous_min_rate; u32 divider; + bool min_rate_supported; + bool max_rate_supported; int err = 0; if (!ESW_ALLOWED(esw)) return -EPERM; if (!LEGAL_VPORT(esw, vport)) return -EINVAL; + + fw_max_bw_share = MLX5_CAP_QOS(esw->dev, max_tsar_bw_share); + min_rate_supported = MLX5_CAP_QOS(esw->dev, esw_bw_share) && + fw_max_bw_share >= MLX5_MIN_BW_SHARE; + max_rate_supported = MLX5_CAP_QOS(esw->dev, esw_rate_limit); + if ((min_rate && !min_rate_supported) || (max_rate && !max_rate_supported)) return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index f2cfa012315e5..0be3eb86dd84e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -263,10 +263,11 @@ static void nested_down_write_ref_node(struct fs_node *node, } } -static void down_write_ref_node(struct fs_node *node) +static void down_write_ref_node(struct fs_node *node, bool locked) { if (node) { - down_write(&node->lock); + if (!locked) + down_write(&node->lock); refcount_inc(&node->refcount); } } @@ -277,13 +278,14 @@ static void up_read_ref_node(struct fs_node *node) up_read(&node->lock); } -static void up_write_ref_node(struct fs_node *node) +static void up_write_ref_node(struct fs_node *node, bool locked) { refcount_dec(&node->refcount); - up_write(&node->lock); + if (!locked) + up_write(&node->lock); } -static void tree_put_node(struct fs_node *node) +static void tree_put_node(struct fs_node *node, bool locked) { struct fs_node *parent_node = node->parent; @@ -294,27 +296,27 @@ static void tree_put_node(struct fs_node *node) /* Only root namespace doesn't have parent and we just * need to free its node. */ - down_write_ref_node(parent_node); + down_write_ref_node(parent_node, locked); list_del_init(&node->list); if (node->del_sw_func) node->del_sw_func(node); - up_write_ref_node(parent_node); + up_write_ref_node(parent_node, locked); } else { kfree(node); } node = NULL; } if (!node && parent_node) - tree_put_node(parent_node); + tree_put_node(parent_node, locked); } -static int tree_remove_node(struct fs_node *node) +static int tree_remove_node(struct fs_node *node, bool locked) { if (refcount_read(&node->refcount) > 1) { refcount_dec(&node->refcount); return -EEXIST; } - tree_put_node(node); + tree_put_node(node, locked); return 0; } @@ -420,22 +422,34 @@ static void del_sw_flow_table(struct fs_node *node) kfree(ft); } -static void del_sw_hw_rule(struct fs_node *node) +static void modify_fte(struct fs_fte *fte) { struct mlx5_flow_root_namespace *root; - struct mlx5_flow_rule *rule; struct mlx5_flow_table *ft; struct mlx5_flow_group *fg; - struct fs_fte *fte; - int modify_mask; - struct mlx5_core_dev *dev = get_dev(node); + struct mlx5_core_dev *dev; int err; - bool update_fte = false; - fs_get_obj(rule, node); - fs_get_obj(fte, rule->node.parent); fs_get_obj(fg, fte->node.parent); fs_get_obj(ft, fg->node.parent); + dev = get_dev(&fte->node); + + root = find_root(&ft->node); + err = root->cmds->update_fte(dev, ft, fg->id, fte->modify_mask, fte); + if (err) + mlx5_core_warn(dev, + "%s can't del rule fg id=%d fte_index=%d\n", + __func__, fg->id, fte->index); + fte->modify_mask = 0; +} + +static void del_sw_hw_rule(struct fs_node *node) +{ + struct mlx5_flow_rule *rule; + struct fs_fte *fte; + + fs_get_obj(rule, node); + fs_get_obj(fte, rule->node.parent); trace_mlx5_fs_del_rule(rule); if (rule->sw_action == MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO) { mutex_lock(&rule->dest_attr.ft->lock); @@ -445,27 +459,19 @@ static void del_sw_hw_rule(struct fs_node *node) if (rule->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER && --fte->dests_size) { - modify_mask = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION) | - BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS); + fte->modify_mask |= + BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION) | + BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS); fte->action.action &= ~MLX5_FLOW_CONTEXT_ACTION_COUNT; - update_fte = true; goto out; } if ((fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) && --fte->dests_size) { - modify_mask = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST); - update_fte = true; + fte->modify_mask |= + BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST); } out: - root = find_root(&ft->node); - if (update_fte && fte->dests_size) { - err = root->cmds->update_fte(dev, ft, fg->id, modify_mask, fte); - if (err) - mlx5_core_warn(dev, - "%s can't del rule fg id=%d fte_index=%d\n", - __func__, fg->id, fte->index); - } kfree(rule); } @@ -491,6 +497,7 @@ static void del_hw_fte(struct fs_node *node) mlx5_core_warn(dev, "flow steering can't delete fte in index %d of flow group id %d\n", fte->index, fg->id); + node->active = 0; } } @@ -591,7 +598,7 @@ static struct fs_fte *alloc_fte(struct mlx5_flow_table *ft, fte->node.type = FS_TYPE_FLOW_ENTRY; fte->action = *flow_act; - tree_init_node(&fte->node, del_hw_fte, del_sw_fte); + tree_init_node(&fte->node, NULL, del_sw_fte); return fte; } @@ -858,7 +865,7 @@ static int _mlx5_modify_rule_destination(struct mlx5_flow_rule *rule, fs_get_obj(fte, rule->node.parent); if (!(fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST)) return -EINVAL; - down_write_ref_node(&fte->node); + down_write_ref_node(&fte->node, false); fs_get_obj(fg, fte->node.parent); fs_get_obj(ft, fg->node.parent); @@ -866,7 +873,7 @@ static int _mlx5_modify_rule_destination(struct mlx5_flow_rule *rule, root = find_root(&ft->node); err = root->cmds->update_fte(get_dev(&ft->node), ft, fg->id, modify_mask, fte); - up_write_ref_node(&fte->node); + up_write_ref_node(&fte->node, false); return err; } @@ -1016,11 +1023,11 @@ static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespa if (err) goto destroy_ft; ft->node.active = true; - down_write_ref_node(&fs_prio->node); + down_write_ref_node(&fs_prio->node, false); tree_add_node(&ft->node, &fs_prio->node); list_add_flow_table(ft, fs_prio); fs_prio->num_ft++; - up_write_ref_node(&fs_prio->node); + up_write_ref_node(&fs_prio->node, false); mutex_unlock(&root->chain_lock); trace_mlx5_fs_add_ft(ft); return ft; @@ -1114,17 +1121,17 @@ struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft, if (ft->autogroup.active) return ERR_PTR(-EPERM); - down_write_ref_node(&ft->node); + down_write_ref_node(&ft->node, false); fg = alloc_insert_flow_group(ft, match_criteria_enable, match_criteria, start_index, end_index, ft->node.children.prev); - up_write_ref_node(&ft->node); + up_write_ref_node(&ft->node, false); if (IS_ERR(fg)) return fg; err = root->cmds->create_flow_group(dev, ft, fg_in, &fg->id); if (err) { - tree_put_node(&fg->node); + tree_put_node(&fg->node, false); return ERR_PTR(err); } trace_mlx5_fs_add_fg(fg); @@ -1521,10 +1528,10 @@ static void free_match_list(struct match_list_head *head) struct match_list *iter, *match_tmp; list_del(&head->first.list); - tree_put_node(&head->first.g->node); + tree_put_node(&head->first.g->node, false); list_for_each_entry_safe(iter, match_tmp, &head->list, list) { - tree_put_node(&iter->g->node); + tree_put_node(&iter->g->node, false); list_del(&iter->list); kfree(iter); } @@ -1601,11 +1608,16 @@ lookup_fte_locked(struct mlx5_flow_group *g, fte_tmp = NULL; goto out; } + if (!fte_tmp->node.active) { + tree_put_node(&fte_tmp->node, false); + fte_tmp = NULL; + goto out; + } nested_down_write_ref_node(&fte_tmp->node, FS_LOCK_CHILD); out: if (take_write) - up_write_ref_node(&g->node); + up_write_ref_node(&g->node, false); else up_read_ref_node(&g->node); return fte_tmp; @@ -1647,8 +1659,8 @@ search_again_locked: continue; rule = add_rule_fg(g, spec->match_value, flow_act, dest, dest_num, fte_tmp); - up_write_ref_node(&fte_tmp->node); - tree_put_node(&fte_tmp->node); + up_write_ref_node(&fte_tmp->node, false); + tree_put_node(&fte_tmp->node, false); kmem_cache_free(steering->ftes_cache, fte); return rule; } @@ -1684,7 +1696,7 @@ skip_search: err = insert_fte(g, fte); if (err) { - up_write_ref_node(&g->node); + up_write_ref_node(&g->node, false); if (err == -ENOSPC) continue; kmem_cache_free(steering->ftes_cache, fte); @@ -1692,11 +1704,11 @@ skip_search: } nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD); - up_write_ref_node(&g->node); + up_write_ref_node(&g->node, false); rule = add_rule_fg(g, spec->match_value, flow_act, dest, dest_num, fte); - up_write_ref_node(&fte->node); - tree_put_node(&fte->node); + up_write_ref_node(&fte->node, false); + tree_put_node(&fte->node, false); return rule; } rule = ERR_PTR(-ENOENT); @@ -1738,7 +1750,7 @@ search_again_locked: err = build_match_list(&match_head, ft, spec); if (err) { if (take_write) - up_write_ref_node(&ft->node); + up_write_ref_node(&ft->node, false); else up_read_ref_node(&ft->node); return ERR_PTR(err); @@ -1753,7 +1765,7 @@ search_again_locked: if (!IS_ERR(rule) || (PTR_ERR(rule) != -ENOENT && PTR_ERR(rule) != -EAGAIN)) { if (take_write) - up_write_ref_node(&ft->node); + up_write_ref_node(&ft->node, false); return rule; } @@ -1769,12 +1781,12 @@ search_again_locked: g = alloc_auto_flow_group(ft, spec); if (IS_ERR(g)) { rule = ERR_CAST(g); - up_write_ref_node(&ft->node); + up_write_ref_node(&ft->node, false); return rule; } nested_down_write_ref_node(&g->node, FS_LOCK_PARENT); - up_write_ref_node(&ft->node); + up_write_ref_node(&ft->node, false); err = create_auto_flow_group(ft, g); if (err) @@ -1793,17 +1805,17 @@ search_again_locked: } nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD); - up_write_ref_node(&g->node); + up_write_ref_node(&g->node, false); rule = add_rule_fg(g, spec->match_value, flow_act, dest, dest_num, fte); - up_write_ref_node(&fte->node); - tree_put_node(&fte->node); - tree_put_node(&g->node); + up_write_ref_node(&fte->node, false); + tree_put_node(&fte->node, false); + tree_put_node(&g->node, false); return rule; err_release_fg: - up_write_ref_node(&g->node); - tree_put_node(&g->node); + up_write_ref_node(&g->node, false); + tree_put_node(&g->node, false); return ERR_PTR(err); } @@ -1866,10 +1878,33 @@ EXPORT_SYMBOL(mlx5_add_flow_rules); void mlx5_del_flow_rules(struct mlx5_flow_handle *handle) { + struct fs_fte *fte; int i; + /* In order to consolidate the HW changes we lock the FTE for other + * changes, and increase its refcount, in order not to perform the + * "del" functions of the FTE. Will handle them here. + * The removal of the rules is done under locked FTE. + * After removing all the handle's rules, if there are remaining + * rules, it means we just need to modify the FTE in FW, and + * unlock/decrease the refcount we increased before. + * Otherwise, it means the FTE should be deleted. First delete the + * FTE in FW. Then, unlock the FTE, and proceed the tree_put_node of + * the FTE, which will handle the last decrease of the refcount, as + * well as required handling of its parent. + */ + fs_get_obj(fte, handle->rule[0]->node.parent); + down_write_ref_node(&fte->node, false); for (i = handle->num_rules - 1; i >= 0; i--) - tree_remove_node(&handle->rule[i]->node); + tree_remove_node(&handle->rule[i]->node, true); + if (fte->modify_mask && fte->dests_size) { + modify_fte(fte); + up_write_ref_node(&fte->node, false); + } else { + del_hw_fte(&fte->node); + up_write(&fte->node.lock); + tree_put_node(&fte->node, false); + } kfree(handle); } EXPORT_SYMBOL(mlx5_del_flow_rules); @@ -1972,7 +2007,7 @@ int mlx5_destroy_flow_table(struct mlx5_flow_table *ft) mutex_unlock(&root->chain_lock); return err; } - if (tree_remove_node(&ft->node)) + if (tree_remove_node(&ft->node, false)) mlx5_core_warn(get_dev(&ft->node), "Flow table %d wasn't destroyed, refcount > 1\n", ft->id); mutex_unlock(&root->chain_lock); @@ -1983,7 +2018,7 @@ EXPORT_SYMBOL(mlx5_destroy_flow_table); void mlx5_destroy_flow_group(struct mlx5_flow_group *fg) { - if (tree_remove_node(&fg->node)) + if (tree_remove_node(&fg->node, false)) mlx5_core_warn(get_dev(&fg->node), "Flow group %d wasn't destroyed, refcount > 1\n", fg->id); } @@ -2367,8 +2402,8 @@ static void clean_tree(struct fs_node *node) tree_get_node(node); list_for_each_entry_safe(iter, temp, &node->children, list) clean_tree(iter); - tree_put_node(node); - tree_remove_node(node); + tree_put_node(node, false); + tree_remove_node(node, false); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h index 2dc86347af587..87de0e4d9124e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h @@ -172,6 +172,7 @@ struct fs_fte { enum fs_fte_status status; struct mlx5_fc *counter; struct rhash_head hash; + int modify_mask; }; /* Type of children is mlx5_flow_table/namespace */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag.c index 48aa6e030bcff..959605559858e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag.c @@ -595,27 +595,6 @@ void mlx5_lag_add(struct mlx5_core_dev *dev, struct net_device *netdev) err); } -int mlx5_lag_get_pf_num(struct mlx5_core_dev *dev, int *pf_num) -{ - struct mlx5_lag *ldev; - int n; - - ldev = mlx5_lag_dev_get(dev); - if (!ldev) { - mlx5_core_warn(dev, "no lag device, can't get pf num\n"); - return -EINVAL; - } - - for (n = 0; n < MLX5_MAX_PORTS; n++) - if (ldev->pf[n].dev == dev) { - *pf_num = n; - return 0; - } - - mlx5_core_warn(dev, "wasn't able to locate pf in the lag device\n"); - return -EINVAL; -} - /* Must be called with intf_mutex held */ void mlx5_lag_remove(struct mlx5_core_dev *dev) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 9529cf9623e31..7b331674622c1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -188,8 +188,6 @@ static inline int mlx5_lag_is_lacp_owner(struct mlx5_core_dev *dev) MLX5_CAP_GEN(dev, lag_master); } -int mlx5_lag_get_pf_num(struct mlx5_core_dev *dev, int *pf_num); - void mlx5_reload_interface(struct mlx5_core_dev *mdev, int protocol); void mlx5_lag_update(struct mlx5_core_dev *dev); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c index 0b85c7252f9e4..472f63f9fac50 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c @@ -111,7 +111,6 @@ struct mlxsw_thermal { struct mlxsw_thermal_trip trips[MLXSW_THERMAL_NUM_TRIPS]; enum thermal_device_mode mode; struct mlxsw_thermal_module *tz_module_arr; - unsigned int tz_module_num; }; static inline u8 mlxsw_state_to_duty(int state) @@ -711,6 +710,9 @@ mlxsw_thermal_module_init(struct device *dev, struct mlxsw_core *core, module = mlxsw_reg_pmlp_module_get(pmlp_pl, 0); module_tz = &thermal->tz_module_arr[module]; + /* Skip if parent is already set (case of port split). */ + if (module_tz->parent) + return 0; module_tz->module = module; module_tz->parent = thermal; memcpy(module_tz->trips, default_thermal_trips, @@ -718,13 +720,7 @@ mlxsw_thermal_module_init(struct device *dev, struct mlxsw_core *core, /* Initialize all trip point. */ mlxsw_thermal_module_trips_reset(module_tz); /* Update trip point according to the module data. */ - err = mlxsw_thermal_module_trips_update(dev, core, module_tz); - if (err) - return err; - - thermal->tz_module_num++; - - return 0; + return mlxsw_thermal_module_trips_update(dev, core, module_tz); } static void mlxsw_thermal_module_fini(struct mlxsw_thermal_module *module_tz) @@ -732,6 +728,7 @@ static void mlxsw_thermal_module_fini(struct mlxsw_thermal_module *module_tz) if (module_tz && module_tz->tzdev) { mlxsw_thermal_module_tz_fini(module_tz->tzdev); module_tz->tzdev = NULL; + module_tz->parent = NULL; } } @@ -740,6 +737,7 @@ mlxsw_thermal_modules_init(struct device *dev, struct mlxsw_core *core, struct mlxsw_thermal *thermal) { unsigned int module_count = mlxsw_core_max_ports(core); + struct mlxsw_thermal_module *module_tz; int i, err; thermal->tz_module_arr = kcalloc(module_count, @@ -754,8 +752,11 @@ mlxsw_thermal_modules_init(struct device *dev, struct mlxsw_core *core, goto err_unreg_tz_module_arr; } - for (i = 0; i < thermal->tz_module_num; i++) { - err = mlxsw_thermal_module_tz_init(&thermal->tz_module_arr[i]); + for (i = 0; i < module_count - 1; i++) { + module_tz = &thermal->tz_module_arr[i]; + if (!module_tz->parent) + continue; + err = mlxsw_thermal_module_tz_init(module_tz); if (err) goto err_unreg_tz_module_arr; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/minimal.c b/drivers/net/ethernet/mellanox/mlxsw/minimal.c index 68bee9572a1b0..00c390024350d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/minimal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/minimal.c @@ -34,6 +34,18 @@ struct mlxsw_m_port { u8 module; }; +static int mlxsw_m_base_mac_get(struct mlxsw_m *mlxsw_m) +{ + char spad_pl[MLXSW_REG_SPAD_LEN] = {0}; + int err; + + err = mlxsw_reg_query(mlxsw_m->core, MLXSW_REG(spad), spad_pl); + if (err) + return err; + mlxsw_reg_spad_base_mac_memcpy_from(spad_pl, mlxsw_m->base_mac); + return 0; +} + static int mlxsw_m_port_dummy_open_stop(struct net_device *dev) { return 0; @@ -314,6 +326,12 @@ static int mlxsw_m_init(struct mlxsw_core *mlxsw_core, mlxsw_m->core = mlxsw_core; mlxsw_m->bus_info = mlxsw_bus_info; + err = mlxsw_m_base_mac_get(mlxsw_m); + if (err) { + dev_err(mlxsw_m->bus_info->dev, "Failed to get base mac\n"); + return err; + } + err = mlxsw_m_ports_create(mlxsw_m); if (err) { dev_err(mlxsw_m->bus_info->dev, "Failed to create ports\n"); diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index 4d1b4a24907fd..13e6bf13ac4de 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -585,8 +585,7 @@ static int lan743x_intr_open(struct lan743x_adapter *adapter) if (adapter->csr.flags & LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR) { - flags = LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_CLEAR | - LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET | + flags = LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET | LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET | LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR | LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR; @@ -599,12 +598,6 @@ static int lan743x_intr_open(struct lan743x_adapter *adapter) /* map TX interrupt to vector */ int_vec_map1 |= INT_VEC_MAP1_TX_VEC_(index, vector); lan743x_csr_write(adapter, INT_VEC_MAP1, int_vec_map1); - if (flags & - LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_CLEAR) { - int_vec_en_auto_clr |= INT_VEC_EN_(vector); - lan743x_csr_write(adapter, INT_VEC_EN_AUTO_CLR, - int_vec_en_auto_clr); - } /* Remove TX interrupt from shared mask */ intr->vector_list[0].int_mask &= ~int_bit; @@ -1902,7 +1895,17 @@ static int lan743x_rx_next_index(struct lan743x_rx *rx, int index) return ((++index) % rx->ring_size); } -static int lan743x_rx_allocate_ring_element(struct lan743x_rx *rx, int index) +static struct sk_buff *lan743x_rx_allocate_skb(struct lan743x_rx *rx) +{ + int length = 0; + + length = (LAN743X_MAX_FRAME_SIZE + ETH_HLEN + 4 + RX_HEAD_PADDING); + return __netdev_alloc_skb(rx->adapter->netdev, + length, GFP_ATOMIC | GFP_DMA); +} + +static int lan743x_rx_init_ring_element(struct lan743x_rx *rx, int index, + struct sk_buff *skb) { struct lan743x_rx_buffer_info *buffer_info; struct lan743x_rx_descriptor *descriptor; @@ -1911,9 +1914,7 @@ static int lan743x_rx_allocate_ring_element(struct lan743x_rx *rx, int index) length = (LAN743X_MAX_FRAME_SIZE + ETH_HLEN + 4 + RX_HEAD_PADDING); descriptor = &rx->ring_cpu_ptr[index]; buffer_info = &rx->buffer_info[index]; - buffer_info->skb = __netdev_alloc_skb(rx->adapter->netdev, - length, - GFP_ATOMIC | GFP_DMA); + buffer_info->skb = skb; if (!(buffer_info->skb)) return -ENOMEM; buffer_info->dma_ptr = dma_map_single(&rx->adapter->pdev->dev, @@ -2060,8 +2061,19 @@ static int lan743x_rx_process_packet(struct lan743x_rx *rx) /* packet is available */ if (first_index == last_index) { /* single buffer packet */ + struct sk_buff *new_skb = NULL; int packet_length; + new_skb = lan743x_rx_allocate_skb(rx); + if (!new_skb) { + /* failed to allocate next skb. + * Memory is very low. + * Drop this packet and reuse buffer. + */ + lan743x_rx_reuse_ring_element(rx, first_index); + goto process_extension; + } + buffer_info = &rx->buffer_info[first_index]; skb = buffer_info->skb; descriptor = &rx->ring_cpu_ptr[first_index]; @@ -2081,7 +2093,7 @@ static int lan743x_rx_process_packet(struct lan743x_rx *rx) skb_put(skb, packet_length - 4); skb->protocol = eth_type_trans(skb, rx->adapter->netdev); - lan743x_rx_allocate_ring_element(rx, first_index); + lan743x_rx_init_ring_element(rx, first_index, new_skb); } else { int index = first_index; @@ -2094,26 +2106,23 @@ static int lan743x_rx_process_packet(struct lan743x_rx *rx) if (first_index <= last_index) { while ((index >= first_index) && (index <= last_index)) { - lan743x_rx_release_ring_element(rx, - index); - lan743x_rx_allocate_ring_element(rx, - index); + lan743x_rx_reuse_ring_element(rx, + index); index = lan743x_rx_next_index(rx, index); } } else { while ((index >= first_index) || (index <= last_index)) { - lan743x_rx_release_ring_element(rx, - index); - lan743x_rx_allocate_ring_element(rx, - index); + lan743x_rx_reuse_ring_element(rx, + index); index = lan743x_rx_next_index(rx, index); } } } +process_extension: if (extension_index >= 0) { descriptor = &rx->ring_cpu_ptr[extension_index]; buffer_info = &rx->buffer_info[extension_index]; @@ -2290,7 +2299,9 @@ static int lan743x_rx_ring_init(struct lan743x_rx *rx) rx->last_head = 0; for (index = 0; index < rx->ring_size; index++) { - ret = lan743x_rx_allocate_ring_element(rx, index); + struct sk_buff *new_skb = lan743x_rx_allocate_skb(rx); + + ret = lan743x_rx_init_ring_element(rx, index, new_skb); if (ret) goto cleanup; } diff --git a/drivers/net/ethernet/qlogic/qla3xxx.c b/drivers/net/ethernet/qlogic/qla3xxx.c index 10b075bc59596..b61b88cbc0c7d 100644 --- a/drivers/net/ethernet/qlogic/qla3xxx.c +++ b/drivers/net/ethernet/qlogic/qla3xxx.c @@ -3886,6 +3886,12 @@ static int ql3xxx_probe(struct pci_dev *pdev, netif_stop_queue(ndev); qdev->workqueue = create_singlethread_workqueue(ndev->name); + if (!qdev->workqueue) { + unregister_netdev(ndev); + err = -ENOMEM; + goto err_out_iounmap; + } + INIT_DELAYED_WORK(&qdev->reset_work, ql_reset_work); INIT_DELAYED_WORK(&qdev->tx_timeout_work, ql_tx_timeout_work); INIT_DELAYED_WORK(&qdev->link_state_work, ql_link_state_machine_work); diff --git a/drivers/net/ethernet/qlogic/qlge/qlge_main.c b/drivers/net/ethernet/qlogic/qlge/qlge_main.c index 096515c272633..07e1c623048e5 100644 --- a/drivers/net/ethernet/qlogic/qlge/qlge_main.c +++ b/drivers/net/ethernet/qlogic/qlge/qlge_main.c @@ -4681,6 +4681,11 @@ static int ql_init_device(struct pci_dev *pdev, struct net_device *ndev, */ qdev->workqueue = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM, ndev->name); + if (!qdev->workqueue) { + err = -ENOMEM; + goto err_out2; + } + INIT_DELAYED_WORK(&qdev->asic_reset_work, ql_asic_reset_work); INIT_DELAYED_WORK(&qdev->mpi_reset_work, ql_mpi_reset_work); INIT_DELAYED_WORK(&qdev->mpi_work, ql_mpi_work); diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c index 339b2eae21006..e33af371b169e 100644 --- a/drivers/net/ethernet/renesas/sh_eth.c +++ b/drivers/net/ethernet/renesas/sh_eth.c @@ -3181,12 +3181,16 @@ static struct sh_eth_plat_data *sh_eth_parse_dt(struct device *dev) struct device_node *np = dev->of_node; struct sh_eth_plat_data *pdata; const char *mac_addr; + int ret; pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) return NULL; - pdata->phy_interface = of_get_phy_mode(np); + ret = of_get_phy_mode(np); + if (ret < 0) + return NULL; + pdata->phy_interface = ret; mac_addr = of_get_mac_address(np); if (mac_addr) diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c index c883aa89b7ca0..a71c900ca04f5 100644 --- a/drivers/net/ethernet/rocker/rocker_main.c +++ b/drivers/net/ethernet/rocker/rocker_main.c @@ -2805,6 +2805,11 @@ static int rocker_switchdev_event(struct notifier_block *unused, memcpy(&switchdev_work->fdb_info, ptr, sizeof(switchdev_work->fdb_info)); switchdev_work->fdb_info.addr = kzalloc(ETH_ALEN, GFP_ATOMIC); + if (unlikely(!switchdev_work->fdb_info.addr)) { + kfree(switchdev_work); + return NOTIFY_BAD; + } + ether_addr_copy((u8 *)switchdev_work->fdb_info.addr, fdb_info->addr); /* Take a reference on the rocker device */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c index 0f660af01a4b8..195669f550f02 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c @@ -1147,7 +1147,10 @@ static int sun8i_dwmac_probe(struct platform_device *pdev) return ret; } - plat_dat->interface = of_get_phy_mode(dev->of_node); + ret = of_get_phy_mode(dev->of_node); + if (ret < 0) + return -EINVAL; + plat_dat->interface = ret; /* platform data specifying hardware features and callbacks. * hardware features were copied from Allwinner drivers. diff --git a/drivers/net/ppp/pptp.c b/drivers/net/ppp/pptp.c index 8f09edd811e90..50c60550f295d 100644 --- a/drivers/net/ppp/pptp.c +++ b/drivers/net/ppp/pptp.c @@ -532,6 +532,7 @@ static void pptp_sock_destruct(struct sock *sk) pppox_unbind_sock(sk); } skb_queue_purge(&sk->sk_receive_queue); + dst_release(rcu_dereference_protected(sk->sk_dst_cache, 1)); } static int pptp_create(struct net *net, struct socket *sock, int kern) diff --git a/drivers/ntb/hw/intel/ntb_hw_gen1.c b/drivers/ntb/hw/intel/ntb_hw_gen1.c index 2ad263f708da7..bb57ec2390299 100644 --- a/drivers/ntb/hw/intel/ntb_hw_gen1.c +++ b/drivers/ntb/hw/intel/ntb_hw_gen1.c @@ -180,7 +180,7 @@ int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx) return ndev->reg->mw_bar[idx]; } -static inline int ndev_db_addr(struct intel_ntb_dev *ndev, +void ndev_db_addr(struct intel_ntb_dev *ndev, phys_addr_t *db_addr, resource_size_t *db_size, phys_addr_t reg_addr, unsigned long reg) { @@ -196,8 +196,6 @@ static inline int ndev_db_addr(struct intel_ntb_dev *ndev, *db_size = ndev->reg->db_size; dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size); } - - return 0; } u64 ndev_db_read(struct intel_ntb_dev *ndev, @@ -1111,13 +1109,28 @@ int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) ndev->self_reg->db_mask); } -int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, - resource_size_t *db_size) +static int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, + resource_size_t *db_size, u64 *db_data, int db_bit) { + u64 db_bits; struct intel_ntb_dev *ndev = ntb_ndev(ntb); - return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr, + if (unlikely(db_bit >= BITS_PER_LONG_LONG)) + return -EINVAL; + + db_bits = BIT_ULL(db_bit); + + if (unlikely(db_bits & ~ntb_ndev(ntb)->db_valid_mask)) + return -EINVAL; + + ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr, ndev->peer_reg->db_bell); + + if (db_data) + *db_data = db_bits; + + + return 0; } static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) diff --git a/drivers/ntb/hw/intel/ntb_hw_gen1.h b/drivers/ntb/hw/intel/ntb_hw_gen1.h index ad8ec1444436e..544cf5c06f4dc 100644 --- a/drivers/ntb/hw/intel/ntb_hw_gen1.h +++ b/drivers/ntb/hw/intel/ntb_hw_gen1.h @@ -147,6 +147,9 @@ extern struct intel_b2b_addr xeon_b2b_dsd_addr; int ndev_init_isr(struct intel_ntb_dev *ndev, int msix_min, int msix_max, int msix_shift, int total_shift); enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd); +void ndev_db_addr(struct intel_ntb_dev *ndev, + phys_addr_t *db_addr, resource_size_t *db_size, + phys_addr_t reg_addr, unsigned long reg); u64 ndev_db_read(struct intel_ntb_dev *ndev, void __iomem *mmio); int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits, void __iomem *mmio); @@ -166,8 +169,6 @@ int intel_ntb_db_vector_count(struct ntb_dev *ntb); u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector); int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits); int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits); -int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, - resource_size_t *db_size); int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb); int intel_ntb_spad_count(struct ntb_dev *ntb); u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx); diff --git a/drivers/ntb/hw/intel/ntb_hw_gen3.c b/drivers/ntb/hw/intel/ntb_hw_gen3.c index b3fa24778f940..f475b56a3f492 100644 --- a/drivers/ntb/hw/intel/ntb_hw_gen3.c +++ b/drivers/ntb/hw/intel/ntb_hw_gen3.c @@ -532,6 +532,37 @@ static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, return 0; } +int intel_ntb3_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, + resource_size_t *db_size, + u64 *db_data, int db_bit) +{ + phys_addr_t db_addr_base; + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + if (unlikely(db_bit >= BITS_PER_LONG_LONG)) + return -EINVAL; + + if (unlikely(BIT_ULL(db_bit) & ~ntb_ndev(ntb)->db_valid_mask)) + return -EINVAL; + + ndev_db_addr(ndev, &db_addr_base, db_size, ndev->peer_addr, + ndev->peer_reg->db_bell); + + if (db_addr) { + *db_addr = db_addr_base + (db_bit * 4); + dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx db bit %d\n", + *db_addr, db_bit); + } + + if (db_data) { + *db_data = 1; + dev_dbg(&ndev->ntb.pdev->dev, "Peer db data %llx db bit %d\n", + *db_data, db_bit); + } + + return 0; +} + static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); @@ -584,7 +615,7 @@ const struct ntb_dev_ops intel_ntb3_ops = { .db_clear = intel_ntb3_db_clear, .db_set_mask = intel_ntb_db_set_mask, .db_clear_mask = intel_ntb_db_clear_mask, - .peer_db_addr = intel_ntb_peer_db_addr, + .peer_db_addr = intel_ntb3_peer_db_addr, .peer_db_set = intel_ntb3_peer_db_set, .spad_is_unsafe = intel_ntb_spad_is_unsafe, .spad_count = intel_ntb_spad_count, diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c index f2df2d39c65b0..d905d368d28c7 100644 --- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c +++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c @@ -236,6 +236,7 @@ static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx) ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN; iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); iowrite32(0, &ctl->bar_entry[bar].win_size); + iowrite32(0, &ctl->bar_ext_entry[bar].win_size); iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr); } @@ -258,7 +259,9 @@ static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx, ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); - iowrite32(xlate_pos | size, &ctl->bar_entry[bar].win_size); + iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000), + &ctl->bar_entry[bar].win_size); + iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size); iowrite64(sndev->self_partition | addr, &ctl->bar_entry[bar].xlate_addr); } @@ -679,11 +682,16 @@ static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb) static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, - resource_size_t *db_size) + resource_size_t *db_size, + u64 *db_data, + int db_bit) { struct switchtec_ntb *sndev = ntb_sndev(ntb); unsigned long offset; + if (unlikely(db_bit >= BITS_PER_LONG_LONG)) + return -EINVAL; + offset = (unsigned long)sndev->mmio_peer_dbmsg->odb - (unsigned long)sndev->stdev->mmio; @@ -693,6 +701,8 @@ static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb, *db_addr = pci_resource_start(ntb->pdev, 0) + offset; if (db_size) *db_size = sizeof(u32); + if (db_data) + *db_data = BIT_ULL(db_bit) << sndev->db_peer_shift; return 0; } @@ -1025,7 +1035,9 @@ static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx, ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); - iowrite32(xlate_pos | size, &ctl->bar_entry[bar].win_size); + iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000), + &ctl->bar_entry[bar].win_size); + iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size); iowrite64(sndev->peer_partition | addr, &ctl->bar_entry[bar].xlate_addr); } @@ -1092,7 +1104,7 @@ static int crosslink_enum_partition(struct switchtec_ntb *sndev, dev_dbg(&sndev->stdev->dev, "Crosslink BAR%d addr: %llx\n", - i, bar_addr); + i*2, bar_addr); if (bar_addr != bar_space * i) continue; diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index 3bfdb45624088..d4f39ba1d9769 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -144,7 +144,9 @@ struct ntb_transport_qp { struct list_head tx_free_q; spinlock_t ntb_tx_free_q_lock; void __iomem *tx_mw; - dma_addr_t tx_mw_phys; + phys_addr_t tx_mw_phys; + size_t tx_mw_size; + dma_addr_t tx_mw_dma_addr; unsigned int tx_index; unsigned int tx_max_entry; unsigned int tx_max_frame; @@ -862,6 +864,9 @@ static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) if (!nt->link_is_up) cancel_delayed_work_sync(&nt->link_work); + for (i = 0; i < nt->mw_count; i++) + ntb_free_mw(nt, i); + /* The scratchpad registers keep the values if the remote side * goes down, blast them now to give them a sane value the next * time they are accessed @@ -1049,6 +1054,7 @@ static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, tx_size = (unsigned int)mw_size / num_qps_mw; qp_offset = tx_size * (qp_num / mw_count); + qp->tx_mw_size = tx_size; qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset; if (!qp->tx_mw) return -EINVAL; @@ -1644,7 +1650,7 @@ static int ntb_async_tx_submit(struct ntb_transport_qp *qp, dma_cookie_t cookie; device = chan->device; - dest = qp->tx_mw_phys + qp->tx_max_frame * entry->tx_index; + dest = qp->tx_mw_dma_addr + qp->tx_max_frame * entry->tx_index; buff_off = (size_t)buf & ~PAGE_MASK; dest_off = (size_t)dest & ~PAGE_MASK; @@ -1863,6 +1869,18 @@ ntb_transport_create_queue(void *data, struct device *client_dev, qp->rx_dma_chan = NULL; } + if (qp->tx_dma_chan) { + qp->tx_mw_dma_addr = + dma_map_resource(qp->tx_dma_chan->device->dev, + qp->tx_mw_phys, qp->tx_mw_size, + DMA_FROM_DEVICE, 0); + if (dma_mapping_error(qp->tx_dma_chan->device->dev, + qp->tx_mw_dma_addr)) { + qp->tx_mw_dma_addr = 0; + goto err1; + } + } + dev_dbg(&pdev->dev, "Using %s memcpy for TX\n", qp->tx_dma_chan ? "DMA" : "CPU"); @@ -1904,6 +1922,10 @@ err1: qp->rx_alloc_entry = 0; while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) kfree(entry); + if (qp->tx_mw_dma_addr) + dma_unmap_resource(qp->tx_dma_chan->device->dev, + qp->tx_mw_dma_addr, qp->tx_mw_size, + DMA_FROM_DEVICE, 0); if (qp->tx_dma_chan) dma_release_channel(qp->tx_dma_chan); if (qp->rx_dma_chan) @@ -1945,6 +1967,11 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp) */ dma_sync_wait(chan, qp->last_cookie); dmaengine_terminate_all(chan); + + dma_unmap_resource(chan->device->dev, + qp->tx_mw_dma_addr, qp->tx_mw_size, + DMA_FROM_DEVICE, 0); + dma_release_channel(chan); } diff --git a/drivers/nvdimm/e820.c b/drivers/nvdimm/e820.c index 521eaf53a52aa..36be9b6191876 100644 --- a/drivers/nvdimm/e820.c +++ b/drivers/nvdimm/e820.c @@ -47,6 +47,7 @@ static int e820_register_one(struct resource *res, void *data) ndr_desc.res = res; ndr_desc.attr_groups = e820_pmem_region_attribute_groups; ndr_desc.numa_node = e820_range_to_nid(res->start); + ndr_desc.target_node = ndr_desc.numa_node; set_bit(ND_REGION_PAGEMAP, &ndr_desc.flags); if (!nvdimm_pmem_region_create(nvdimm_bus, &ndr_desc)) return -ENXIO; diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index 379bf4305e615..a5ac3b240293b 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -153,7 +153,7 @@ struct nd_region { u16 ndr_mappings; u64 ndr_size; u64 ndr_start; - int id, num_lanes, ro, numa_node; + int id, num_lanes, ro, numa_node, target_node; void *provider_data; struct kernfs_node *bb_state; struct badblocks bb; diff --git a/drivers/nvdimm/of_pmem.c b/drivers/nvdimm/of_pmem.c index 11b9821eba859..a0c8dcfa0bf92 100644 --- a/drivers/nvdimm/of_pmem.c +++ b/drivers/nvdimm/of_pmem.c @@ -68,6 +68,7 @@ static int of_pmem_region_probe(struct platform_device *pdev) memset(&ndr_desc, 0, sizeof(ndr_desc)); ndr_desc.attr_groups = region_attr_groups; ndr_desc.numa_node = dev_to_node(&pdev->dev); + ndr_desc.target_node = ndr_desc.numa_node; ndr_desc.res = &pdev->resource[i]; ndr_desc.of_node = np; set_bit(ND_REGION_PAGEMAP, &ndr_desc.flags); diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index 3b58baa44b5cf..b4ef7d9ff22eb 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -1072,6 +1072,7 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus, nd_region->flags = ndr_desc->flags; nd_region->ro = ro; nd_region->numa_node = ndr_desc->numa_node; + nd_region->target_node = ndr_desc->target_node; ida_init(&nd_region->ns_ida); ida_init(&nd_region->btt_ida); ida_init(&nd_region->pfn_ida); diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 07bf2bff3a76f..470601980794e 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -179,8 +179,8 @@ static int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl) int ret = 0; /* - * Keep a reference until the work is flushed since ->delete_ctrl - * can free the controller. + * Keep a reference until nvme_do_delete_ctrl() complete, + * since ->delete_ctrl can free the controller. */ nvme_get_ctrl(ctrl); if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING)) @@ -1250,7 +1250,7 @@ static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, if (ns) { if (ctrl->effects) effects = le32_to_cpu(ctrl->effects->iocs[opcode]); - if (effects & ~NVME_CMD_EFFECTS_CSUPP) + if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC)) dev_warn(ctrl->device, "IO command:%02x has unhandled effects:%08x\n", opcode, effects); @@ -1495,10 +1495,10 @@ static void nvme_set_chunk_size(struct nvme_ns *ns) blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(chunk_size)); } -static void nvme_config_discard(struct nvme_ns *ns) +static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns) { struct nvme_ctrl *ctrl = ns->ctrl; - struct request_queue *queue = ns->queue; + struct request_queue *queue = disk->queue; u32 size = queue_logical_block_size(queue); if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) { @@ -1526,12 +1526,13 @@ static void nvme_config_discard(struct nvme_ns *ns) blk_queue_max_write_zeroes_sectors(queue, UINT_MAX); } -static inline void nvme_config_write_zeroes(struct nvme_ns *ns) +static void nvme_config_write_zeroes(struct gendisk *disk, struct nvme_ns *ns) { u32 max_sectors; unsigned short bs = 1 << ns->lba_shift; - if (!(ns->ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES)) + if (!(ns->ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) || + (ns->ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES)) return; /* * Even though NVMe spec explicitly states that MDTS is not @@ -1548,13 +1549,7 @@ static inline void nvme_config_write_zeroes(struct nvme_ns *ns) else max_sectors = ((u32)(ns->ctrl->max_hw_sectors + 1) * bs) >> 9; - blk_queue_max_write_zeroes_sectors(ns->queue, max_sectors); -} - -static inline void nvme_ns_config_oncs(struct nvme_ns *ns) -{ - nvme_config_discard(ns); - nvme_config_write_zeroes(ns); + blk_queue_max_write_zeroes_sectors(disk->queue, max_sectors); } static void nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid, @@ -1610,7 +1605,9 @@ static void nvme_update_disk_info(struct gendisk *disk, capacity = 0; set_capacity(disk, capacity); - nvme_ns_config_oncs(ns); + + nvme_config_discard(disk, ns); + nvme_config_write_zeroes(disk, ns); if (id->nsattr & (1 << 0)) set_disk_ro(disk, true); @@ -3304,6 +3301,7 @@ static int nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) mutex_lock(&ctrl->subsys->lock); list_del_rcu(&ns->siblings); mutex_unlock(&ctrl->subsys->lock); + nvme_put_ns_head(ns->head); out_free_id: kfree(id); out_free_queue: diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index b29b12498a1a9..f3b9d91ba0dfd 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2107,7 +2107,7 @@ nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq, freq->sg_cnt = 0; - if (!blk_rq_payload_bytes(rq)) + if (!blk_rq_nr_phys_segments(rq)) return 0; freq->sg_table.sgl = freq->first_sgl; @@ -2304,12 +2304,23 @@ nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx, if (ret) return ret; - data_len = blk_rq_payload_bytes(rq); - if (data_len) + /* + * nvme core doesn't quite treat the rq opaquely. Commands such + * as WRITE ZEROES will return a non-zero rq payload_bytes yet + * there is no actual payload to be transferred. + * To get it right, key data transmission on there being 1 or + * more physical segments in the sg list. If there is no + * physical segments, there is no payload. + */ + if (blk_rq_nr_phys_segments(rq)) { + data_len = blk_rq_payload_bytes(rq); io_dir = ((rq_data_dir(rq) == WRITE) ? NVMEFC_FCP_WRITE : NVMEFC_FCP_READ); - else + } else { + data_len = 0; io_dir = NVMEFC_FCP_NODATA; + } + return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir); } @@ -2464,6 +2475,7 @@ static int nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl) { struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; + u32 prior_ioq_cnt = ctrl->ctrl.queue_count - 1; unsigned int nr_io_queues; int ret; @@ -2476,6 +2488,13 @@ nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl) return ret; } + if (!nr_io_queues && prior_ioq_cnt) { + dev_info(ctrl->ctrl.device, + "Fail Reconnect: At least 1 io queue " + "required (was %d)\n", prior_ioq_cnt); + return -ENOSPC; + } + ctrl->ctrl.queue_count = nr_io_queues + 1; /* check for io queues existing */ if (ctrl->ctrl.queue_count == 1) @@ -2489,6 +2508,10 @@ nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl) if (ret) goto out_delete_hw_queues; + if (prior_ioq_cnt != nr_io_queues) + dev_info(ctrl->ctrl.device, + "reconnect: revising io queue count from %d to %d\n", + prior_ioq_cnt, nr_io_queues); blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues); return 0; @@ -3006,7 +3029,10 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, ctrl->ctrl.opts = opts; ctrl->ctrl.nr_reconnects = 0; - ctrl->ctrl.numa_node = dev_to_node(lport->dev); + if (lport->dev) + ctrl->ctrl.numa_node = dev_to_node(lport->dev); + else + ctrl->ctrl.numa_node = NUMA_NO_NODE; INIT_LIST_HEAD(&ctrl->ctrl_list); ctrl->lport = lport; ctrl->rport = rport; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index b91f1838bbd5d..527d645450230 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -87,6 +87,11 @@ enum nvme_quirks { * Ignore device provided subnqn. */ NVME_QUIRK_IGNORE_DEV_SUBNQN = (1 << 8), + + /* + * Broken Write Zeroes. + */ + NVME_QUIRK_DISABLE_WRITE_ZEROES = (1 << 9), }; /* diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 92bad1c810acd..a90cf5d63aac4 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -2937,7 +2937,8 @@ static const struct pci_device_id nvme_id_table[] = { { PCI_VDEVICE(INTEL, 0xf1a6), /* Intel 760p/Pro 7600p */ .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN, }, { PCI_VDEVICE(INTEL, 0x5845), /* Qemu emulated controller */ - .driver_data = NVME_QUIRK_IDENTIFY_CNS, }, + .driver_data = NVME_QUIRK_IDENTIFY_CNS | + NVME_QUIRK_DISABLE_WRITE_ZEROES, }, { PCI_DEVICE(0x1bb1, 0x0100), /* Seagate Nytro Flash Storage */ .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, }, { PCI_DEVICE(0x1c58, 0x0003), /* HGST adapter */ diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 208ee518af650..e7e08889865e7 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -463,6 +463,15 @@ static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, queue->data_remaining = le32_to_cpu(pdu->data_length); + if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS && + unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) { + dev_err(queue->ctrl->ctrl.device, + "queue %d tag %#x SUCCESS set but not last PDU\n", + nvme_tcp_queue_id(queue), rq->tag); + nvme_tcp_error_recovery(&queue->ctrl->ctrl); + return -EPROTO; + } + return 0; } @@ -618,6 +627,14 @@ static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb, return ret; } +static inline void nvme_tcp_end_request(struct request *rq, __le16 status) +{ + union nvme_result res = {}; + + nvme_end_request(rq, cpu_to_le16(status << 1), res); +} + + static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, unsigned int *offset, size_t *len) { @@ -685,6 +702,8 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst); queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; } else { + if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) + nvme_tcp_end_request(rq, NVME_SC_SUCCESS); nvme_tcp_init_recv_ctx(queue); } } @@ -695,6 +714,7 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, struct sk_buff *skb, unsigned int *offset, size_t *len) { + struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; char *ddgst = (char *)&queue->recv_ddgst; size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining); off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining; @@ -718,6 +738,13 @@ static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, return -EIO; } + if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { + struct request *rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), + pdu->command_id); + + nvme_tcp_end_request(rq, NVME_SC_SUCCESS); + } + nvme_tcp_init_recv_ctx(queue); return 0; } @@ -815,10 +842,7 @@ static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue) static void nvme_tcp_fail_request(struct nvme_tcp_request *req) { - union nvme_result res = {}; - - nvme_end_request(blk_mq_rq_from_pdu(req), - cpu_to_le16(NVME_SC_DATA_XFER_ERROR), res); + nvme_tcp_end_request(blk_mq_rq_from_pdu(req), NVME_SC_DATA_XFER_ERROR); } static int nvme_tcp_try_send_data(struct nvme_tcp_request *req) diff --git a/drivers/nvme/host/trace.c b/drivers/nvme/host/trace.c index 58456de78bb2a..5f24ea7a28eb8 100644 --- a/drivers/nvme/host/trace.c +++ b/drivers/nvme/host/trace.c @@ -50,7 +50,19 @@ static const char *nvme_trace_admin_identify(struct trace_seq *p, u8 *cdw10) return ret; } +static const char *nvme_trace_admin_get_features(struct trace_seq *p, + u8 *cdw10) +{ + const char *ret = trace_seq_buffer_ptr(p); + u8 fid = cdw10[0]; + u8 sel = cdw10[1] & 0x7; + u32 cdw11 = get_unaligned_le32(cdw10 + 4); + + trace_seq_printf(p, "fid=0x%x sel=0x%x cdw11=0x%x", fid, sel, cdw11); + trace_seq_putc(p, 0); + return ret; +} static const char *nvme_trace_read_write(struct trace_seq *p, u8 *cdw10) { @@ -101,6 +113,8 @@ const char *nvme_trace_parse_admin_cmd(struct trace_seq *p, return nvme_trace_create_cq(p, cdw10); case nvme_admin_identify: return nvme_trace_admin_identify(p, cdw10); + case nvme_admin_get_features: + return nvme_trace_admin_get_features(p, cdw10); default: return nvme_trace_common(p, cdw10); } diff --git a/drivers/nvme/host/trace.h b/drivers/nvme/host/trace.h index 244d7c177e5a7..97d3c77365b89 100644 --- a/drivers/nvme/host/trace.h +++ b/drivers/nvme/host/trace.h @@ -108,7 +108,7 @@ TRACE_EVENT(nvme_setup_cmd, __entry->metadata = le64_to_cpu(cmd->common.metadata); __assign_disk_name(__entry->disk, req->rq_disk); memcpy(__entry->cdw10, &cmd->common.cdw10, - 6 * sizeof(__entry->cdw10)); + sizeof(__entry->cdw10)); ), TP_printk("nvme%d: %sqid=%d, cmdid=%u, nsid=%u, flags=0x%x, meta=0x%llx, cmd=(%s %s)", __entry->ctrl_id, __print_disk_name(__entry->disk), diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index d44ede147263b..2d73b66e36862 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1163,6 +1163,15 @@ static void nvmet_release_p2p_ns_map(struct nvmet_ctrl *ctrl) put_device(ctrl->p2p_client); } +static void nvmet_fatal_error_handler(struct work_struct *work) +{ + struct nvmet_ctrl *ctrl = + container_of(work, struct nvmet_ctrl, fatal_err_work); + + pr_err("ctrl %d fatal error occurred!\n", ctrl->cntlid); + ctrl->ops->delete_ctrl(ctrl); +} + u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn, struct nvmet_req *req, u32 kato, struct nvmet_ctrl **ctrlp) { @@ -1205,6 +1214,7 @@ u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn, INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work); INIT_LIST_HEAD(&ctrl->async_events); INIT_RADIX_TREE(&ctrl->p2p_ns_map, GFP_KERNEL); + INIT_WORK(&ctrl->fatal_err_work, nvmet_fatal_error_handler); memcpy(ctrl->subsysnqn, subsysnqn, NVMF_NQN_SIZE); memcpy(ctrl->hostnqn, hostnqn, NVMF_NQN_SIZE); @@ -1308,21 +1318,11 @@ void nvmet_ctrl_put(struct nvmet_ctrl *ctrl) kref_put(&ctrl->ref, nvmet_ctrl_free); } -static void nvmet_fatal_error_handler(struct work_struct *work) -{ - struct nvmet_ctrl *ctrl = - container_of(work, struct nvmet_ctrl, fatal_err_work); - - pr_err("ctrl %d fatal error occurred!\n", ctrl->cntlid); - ctrl->ops->delete_ctrl(ctrl); -} - void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl) { mutex_lock(&ctrl->lock); if (!(ctrl->csts & NVME_CSTS_CFS)) { ctrl->csts |= NVME_CSTS_CFS; - INIT_WORK(&ctrl->fatal_err_work, nvmet_fatal_error_handler); schedule_work(&ctrl->fatal_err_work); } mutex_unlock(&ctrl->lock); diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 1e9654f04c608..98b7b1f4ee96a 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -1143,10 +1143,8 @@ __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport) &tgtport->assoc_list, a_list) { if (!nvmet_fc_tgt_a_get(assoc)) continue; - spin_unlock_irqrestore(&tgtport->lock, flags); - nvmet_fc_delete_target_assoc(assoc); - nvmet_fc_tgt_a_put(assoc); - spin_lock_irqsave(&tgtport->lock, flags); + if (!schedule_work(&assoc->del_work)) + nvmet_fc_tgt_a_put(assoc); } spin_unlock_irqrestore(&tgtport->lock, flags); } @@ -1185,7 +1183,8 @@ nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl) nvmet_fc_tgtport_put(tgtport); if (found_ctrl) { - schedule_work(&assoc->del_work); + if (!schedule_work(&assoc->del_work)) + nvmet_fc_tgt_a_put(assoc); return; } @@ -1503,10 +1502,8 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, (struct fcnvme_ls_disconnect_rqst *)iod->rqstbuf; struct fcnvme_ls_disconnect_acc *acc = (struct fcnvme_ls_disconnect_acc *)iod->rspbuf; - struct nvmet_fc_tgt_queue *queue = NULL; struct nvmet_fc_tgt_assoc *assoc; int ret = 0; - bool del_assoc = false; memset(acc, 0, sizeof(*acc)); @@ -1537,18 +1534,7 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, assoc = nvmet_fc_find_target_assoc(tgtport, be64_to_cpu(rqst->associd.association_id)); iod->assoc = assoc; - if (assoc) { - if (rqst->discon_cmd.scope == - FCNVME_DISCONN_CONNECTION) { - queue = nvmet_fc_find_target_queue(tgtport, - be64_to_cpu( - rqst->discon_cmd.id)); - if (!queue) { - nvmet_fc_tgt_a_put(assoc); - ret = VERR_NO_CONN; - } - } - } else + if (!assoc) ret = VERR_NO_ASSOC; } @@ -1576,26 +1562,10 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, sizeof(struct fcnvme_ls_disconnect_acc)), FCNVME_LS_DISCONNECT); - - /* are we to delete a Connection ID (queue) */ - if (queue) { - int qid = queue->qid; - - nvmet_fc_delete_target_queue(queue); - - /* release the get taken by find_target_queue */ - nvmet_fc_tgt_q_put(queue); - - /* tear association down if io queue terminated */ - if (!qid) - del_assoc = true; - } - /* release get taken in nvmet_fc_find_target_assoc */ nvmet_fc_tgt_a_put(iod->assoc); - if (del_assoc) - nvmet_fc_delete_target_assoc(iod->assoc); + nvmet_fc_delete_target_assoc(iod->assoc); } diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 71dfedbadc262..a065dbfc43b19 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -194,11 +194,11 @@ static u16 nvmet_bdev_discard_range(struct nvmet_req *req, le64_to_cpu(range->slba) << (ns->blksize_shift - 9), le32_to_cpu(range->nlb) << (ns->blksize_shift - 9), GFP_KERNEL, 0, bio); - - if (ret) + if (ret && ret != -EOPNOTSUPP) { req->error_slba = le64_to_cpu(range->slba); - - return blk_to_nvme_status(req, errno_to_blk_status(ret)); + return blk_to_nvme_status(req, errno_to_blk_status(ret)); + } + return NVME_SC_SUCCESS; } static void nvmet_bdev_execute_discard(struct nvmet_req *req) diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c index 517522305e5cb..3e43212d3c1c6 100644 --- a/drivers/nvme/target/io-cmd-file.c +++ b/drivers/nvme/target/io-cmd-file.c @@ -297,7 +297,7 @@ static void nvmet_file_execute_discard(struct nvmet_req *req) } ret = vfs_fallocate(req->ns->file, mode, offset, len); - if (ret) { + if (ret && ret != -EOPNOTSUPP) { req->error_slba = le64_to_cpu(range.slba); status = errno_to_nvme_status(req, ret); break; diff --git a/drivers/opp/core.c b/drivers/opp/core.c index d7f97167cac31..0420f7e8ad5b0 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -760,7 +760,7 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) old_freq, freq); /* Scaling up? Configure required OPPs before frequency */ - if (freq > old_freq) { + if (freq >= old_freq) { ret = _set_required_opps(dev, opp_table, opp); if (ret) goto put_opp; diff --git a/drivers/opp/of.c b/drivers/opp/of.c index 62504b18f1985..c10c782d15aa6 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -173,7 +173,7 @@ static void _opp_table_alloc_required_tables(struct opp_table *opp_table, struct opp_table **required_opp_tables; struct device **genpd_virt_devs = NULL; struct device_node *required_np, *np; - int count, i; + int count, count_pd, i; /* Traversing the first OPP node is all we need */ np = of_get_next_available_child(opp_np, NULL); @@ -186,7 +186,19 @@ static void _opp_table_alloc_required_tables(struct opp_table *opp_table, if (!count) goto put_np; - if (count > 1) { + /* + * Check the number of power-domains to know if we need to deal + * with virtual devices. In some cases we have devices with multiple + * power domains but with only one of them being scalable, hence + * 'count' could be 1, but we still have to deal with multiple genpds + * and virtual devices. + */ + count_pd = of_count_phandle_with_args(dev->of_node, "power-domains", + "#power-domain-cells"); + if (!count_pd) + goto put_np; + + if (count_pd > 1) { genpd_virt_devs = kcalloc(count, sizeof(*genpd_virt_devs), GFP_KERNEL); if (!genpd_virt_devs) diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c index 79374d1de3117..1f3ef9ee493c0 100644 --- a/drivers/remoteproc/qcom_q6v5_adsp.c +++ b/drivers/remoteproc/qcom_q6v5_adsp.c @@ -48,7 +48,7 @@ /* list of clocks required by ADSP PIL */ static const char * const adsp_clk_id[] = { - "sway_cbcr", "lpass_aon", "lpass_ahbs_aon_cbcr", "lpass_ahbm_aon_cbcr", + "sway_cbcr", "lpass_ahbs_aon_cbcr", "lpass_ahbm_aon_cbcr", "qdsp6ss_xo", "qdsp6ss_sleep", "qdsp6ss_core", }; @@ -439,6 +439,10 @@ static int adsp_probe(struct platform_device *pdev) adsp->sysmon = qcom_add_sysmon_subdev(rproc, desc->sysmon_name, desc->ssctl_id); + if (IS_ERR(adsp->sysmon)) { + ret = PTR_ERR(adsp->sysmon); + goto disable_pm; + } ret = rproc_add(rproc); if (ret) diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index 01be7314e176f..eacdf10fcfaf3 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -25,6 +25,8 @@ #include <linux/of_address.h> #include <linux/of_device.h> #include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/pm_runtime.h> #include <linux/regmap.h> #include <linux/regulator/consumer.h> #include <linux/remoteproc.h> @@ -131,6 +133,8 @@ struct rproc_hexagon_res { char **proxy_clk_names; char **reset_clk_names; char **active_clk_names; + char **active_pd_names; + char **proxy_pd_names; int version; bool need_mem_protection; bool has_alt_reset; @@ -156,9 +160,13 @@ struct q6v5 { struct clk *active_clks[8]; struct clk *reset_clks[4]; struct clk *proxy_clks[4]; + struct device *active_pds[1]; + struct device *proxy_pds[3]; int active_clk_count; int reset_clk_count; int proxy_clk_count; + int active_pd_count; + int proxy_pd_count; struct reg_info active_regs[1]; struct reg_info proxy_regs[3]; @@ -188,6 +196,7 @@ struct q6v5 { bool has_alt_reset; int mpss_perm; int mba_perm; + const char *hexagon_mdt_image; int version; }; @@ -321,6 +330,41 @@ static void q6v5_clk_disable(struct device *dev, clk_disable_unprepare(clks[i]); } +static int q6v5_pds_enable(struct q6v5 *qproc, struct device **pds, + size_t pd_count) +{ + int ret; + int i; + + for (i = 0; i < pd_count; i++) { + dev_pm_genpd_set_performance_state(pds[i], INT_MAX); + ret = pm_runtime_get_sync(pds[i]); + if (ret < 0) + goto unroll_pd_votes; + } + + return 0; + +unroll_pd_votes: + for (i--; i >= 0; i--) { + dev_pm_genpd_set_performance_state(pds[i], 0); + pm_runtime_put(pds[i]); + } + + return ret; +}; + +static void q6v5_pds_disable(struct q6v5 *qproc, struct device **pds, + size_t pd_count) +{ + int i; + + for (i = 0; i < pd_count; i++) { + dev_pm_genpd_set_performance_state(pds[i], 0); + pm_runtime_put(pds[i]); + } +} + static int q6v5_xfer_mem_ownership(struct q6v5 *qproc, int *current_perm, bool remote_owner, phys_addr_t addr, size_t size) @@ -690,11 +734,23 @@ static int q6v5_mba_load(struct q6v5 *qproc) qcom_q6v5_prepare(&qproc->q6v5); + ret = q6v5_pds_enable(qproc, qproc->active_pds, qproc->active_pd_count); + if (ret < 0) { + dev_err(qproc->dev, "failed to enable active power domains\n"); + goto disable_irqs; + } + + ret = q6v5_pds_enable(qproc, qproc->proxy_pds, qproc->proxy_pd_count); + if (ret < 0) { + dev_err(qproc->dev, "failed to enable proxy power domains\n"); + goto disable_active_pds; + } + ret = q6v5_regulator_enable(qproc, qproc->proxy_regs, qproc->proxy_reg_count); if (ret) { dev_err(qproc->dev, "failed to enable proxy supplies\n"); - goto disable_irqs; + goto disable_proxy_pds; } ret = q6v5_clk_enable(qproc->dev, qproc->proxy_clks, @@ -791,6 +847,10 @@ disable_proxy_clk: disable_proxy_reg: q6v5_regulator_disable(qproc, qproc->proxy_regs, qproc->proxy_reg_count); +disable_proxy_pds: + q6v5_pds_disable(qproc, qproc->proxy_pds, qproc->proxy_pd_count); +disable_active_pds: + q6v5_pds_disable(qproc, qproc->active_pds, qproc->active_pd_count); disable_irqs: qcom_q6v5_unprepare(&qproc->q6v5); @@ -830,6 +890,7 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc) qproc->active_clk_count); q6v5_regulator_disable(qproc, qproc->active_regs, qproc->active_reg_count); + q6v5_pds_disable(qproc, qproc->active_pds, qproc->active_pd_count); /* In case of failure or coredump scenario where reclaiming MBA memory * could not happen reclaim it here. @@ -841,6 +902,8 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc) ret = qcom_q6v5_unprepare(&qproc->q6v5); if (ret) { + q6v5_pds_disable(qproc, qproc->proxy_pds, + qproc->proxy_pd_count); q6v5_clk_disable(qproc->dev, qproc->proxy_clks, qproc->proxy_clk_count); q6v5_regulator_disable(qproc, qproc->proxy_regs, @@ -860,17 +923,26 @@ static int q6v5_mpss_load(struct q6v5 *qproc) phys_addr_t min_addr = PHYS_ADDR_MAX; phys_addr_t max_addr = 0; bool relocate = false; - char seg_name[10]; + char *fw_name; + size_t fw_name_len; ssize_t offset; size_t size = 0; void *ptr; int ret; int i; - ret = request_firmware(&fw, "modem.mdt", qproc->dev); + fw_name_len = strlen(qproc->hexagon_mdt_image); + if (fw_name_len <= 4) + return -EINVAL; + + fw_name = kstrdup(qproc->hexagon_mdt_image, GFP_KERNEL); + if (!fw_name) + return -ENOMEM; + + ret = request_firmware(&fw, fw_name, qproc->dev); if (ret < 0) { - dev_err(qproc->dev, "unable to load modem.mdt\n"); - return ret; + dev_err(qproc->dev, "unable to load %s\n", fw_name); + goto out; } /* Initialize the RMB validator */ @@ -918,10 +990,11 @@ static int q6v5_mpss_load(struct q6v5 *qproc) ptr = qproc->mpss_region + offset; if (phdr->p_filesz) { - snprintf(seg_name, sizeof(seg_name), "modem.b%02d", i); - ret = request_firmware(&seg_fw, seg_name, qproc->dev); + /* Replace "xxx.xxx" with "xxx.bxx" */ + sprintf(fw_name + fw_name_len - 3, "b%02d", i); + ret = request_firmware(&seg_fw, fw_name, qproc->dev); if (ret) { - dev_err(qproc->dev, "failed to load %s\n", seg_name); + dev_err(qproc->dev, "failed to load %s\n", fw_name); goto release_firmware; } @@ -960,6 +1033,8 @@ static int q6v5_mpss_load(struct q6v5 *qproc) release_firmware: release_firmware(fw); +out: + kfree(fw_name); return ret < 0 ? ret : 0; } @@ -1075,9 +1150,10 @@ static int qcom_q6v5_register_dump_segments(struct rproc *rproc, unsigned long i; int ret; - ret = request_firmware(&fw, "modem.mdt", qproc->dev); + ret = request_firmware(&fw, qproc->hexagon_mdt_image, qproc->dev); if (ret < 0) { - dev_err(qproc->dev, "unable to load modem.mdt\n"); + dev_err(qproc->dev, "unable to load %s\n", + qproc->hexagon_mdt_image); return ret; } @@ -1121,6 +1197,7 @@ static void qcom_msa_handover(struct qcom_q6v5 *q6v5) qproc->proxy_clk_count); q6v5_regulator_disable(qproc, qproc->proxy_regs, qproc->proxy_reg_count); + q6v5_pds_disable(qproc, qproc->proxy_pds, qproc->proxy_pd_count); } static int q6v5_init_mem(struct q6v5 *qproc, struct platform_device *pdev) @@ -1181,6 +1258,45 @@ static int q6v5_init_clocks(struct device *dev, struct clk **clks, return i; } +static int q6v5_pds_attach(struct device *dev, struct device **devs, + char **pd_names) +{ + size_t num_pds = 0; + int ret; + int i; + + if (!pd_names) + return 0; + + while (pd_names[num_pds]) + num_pds++; + + for (i = 0; i < num_pds; i++) { + devs[i] = dev_pm_domain_attach_by_name(dev, pd_names[i]); + if (IS_ERR(devs[i])) { + ret = PTR_ERR(devs[i]); + goto unroll_attach; + } + } + + return num_pds; + +unroll_attach: + for (i--; i >= 0; i--) + dev_pm_domain_detach(devs[i], false); + + return ret; +}; + +static void q6v5_pds_detach(struct q6v5 *qproc, struct device **pds, + size_t pd_count) +{ + int i; + + for (i = 0; i < pd_count; i++) + dev_pm_domain_detach(pds[i], false); +} + static int q6v5_init_reset(struct q6v5 *qproc) { qproc->mss_restart = devm_reset_control_get_exclusive(qproc->dev, @@ -1253,6 +1369,7 @@ static int q6v5_probe(struct platform_device *pdev) const struct rproc_hexagon_res *desc; struct q6v5 *qproc; struct rproc *rproc; + const char *mba_image; int ret; desc = of_device_get_match_data(&pdev->dev); @@ -1262,16 +1379,30 @@ static int q6v5_probe(struct platform_device *pdev) if (desc->need_mem_protection && !qcom_scm_is_available()) return -EPROBE_DEFER; + mba_image = desc->hexagon_mba_image; + ret = of_property_read_string_index(pdev->dev.of_node, "firmware-name", + 0, &mba_image); + if (ret < 0 && ret != -EINVAL) + return ret; + rproc = rproc_alloc(&pdev->dev, pdev->name, &q6v5_ops, - desc->hexagon_mba_image, sizeof(*qproc)); + mba_image, sizeof(*qproc)); if (!rproc) { dev_err(&pdev->dev, "failed to allocate rproc\n"); return -ENOMEM; } + rproc->auto_boot = false; + qproc = (struct q6v5 *)rproc->priv; qproc->dev = &pdev->dev; qproc->rproc = rproc; + qproc->hexagon_mdt_image = "modem.mdt"; + ret = of_property_read_string_index(pdev->dev.of_node, "firmware-name", + 1, &qproc->hexagon_mdt_image); + if (ret < 0 && ret != -EINVAL) + return ret; + platform_set_drvdata(pdev, qproc); ret = q6v5_init_mem(qproc, pdev); @@ -1322,10 +1453,26 @@ static int q6v5_probe(struct platform_device *pdev) } qproc->active_reg_count = ret; + ret = q6v5_pds_attach(&pdev->dev, qproc->active_pds, + desc->active_pd_names); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to attach active power domains\n"); + goto free_rproc; + } + qproc->active_pd_count = ret; + + ret = q6v5_pds_attach(&pdev->dev, qproc->proxy_pds, + desc->proxy_pd_names); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to init power domains\n"); + goto detach_active_pds; + } + qproc->proxy_pd_count = ret; + qproc->has_alt_reset = desc->has_alt_reset; ret = q6v5_init_reset(qproc); if (ret) - goto free_rproc; + goto detach_proxy_pds; qproc->version = desc->version; qproc->need_mem_protection = desc->need_mem_protection; @@ -1333,7 +1480,7 @@ static int q6v5_probe(struct platform_device *pdev) ret = qcom_q6v5_init(&qproc->q6v5, pdev, rproc, MPSS_CRASH_REASON_SMEM, qcom_msa_handover); if (ret) - goto free_rproc; + goto detach_proxy_pds; qproc->mpss_perm = BIT(QCOM_SCM_VMID_HLOS); qproc->mba_perm = BIT(QCOM_SCM_VMID_HLOS); @@ -1341,13 +1488,21 @@ static int q6v5_probe(struct platform_device *pdev) qcom_add_smd_subdev(rproc, &qproc->smd_subdev); qcom_add_ssr_subdev(rproc, &qproc->ssr_subdev, "mpss"); qproc->sysmon = qcom_add_sysmon_subdev(rproc, "modem", 0x12); + if (IS_ERR(qproc->sysmon)) { + ret = PTR_ERR(qproc->sysmon); + goto detach_proxy_pds; + } ret = rproc_add(rproc); if (ret) - goto free_rproc; + goto detach_proxy_pds; return 0; +detach_proxy_pds: + q6v5_pds_detach(qproc, qproc->proxy_pds, qproc->proxy_pd_count); +detach_active_pds: + q6v5_pds_detach(qproc, qproc->active_pds, qproc->active_pd_count); free_rproc: rproc_free(rproc); @@ -1364,6 +1519,10 @@ static int q6v5_remove(struct platform_device *pdev) qcom_remove_glink_subdev(qproc->rproc, &qproc->glink_subdev); qcom_remove_smd_subdev(qproc->rproc, &qproc->smd_subdev); qcom_remove_ssr_subdev(qproc->rproc, &qproc->ssr_subdev); + + q6v5_pds_detach(qproc, qproc->active_pds, qproc->active_pd_count); + q6v5_pds_detach(qproc, qproc->proxy_pds, qproc->proxy_pd_count); + rproc_free(qproc->rproc); return 0; @@ -1388,6 +1547,16 @@ static const struct rproc_hexagon_res sdm845_mss = { "mnoc_axi", NULL }, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "cx", + "mx", + "mss", + NULL + }, .need_mem_protection = true, .has_alt_reset = true, .version = MSS_SDM845, @@ -1395,16 +1564,26 @@ static const struct rproc_hexagon_res sdm845_mss = { static const struct rproc_hexagon_res msm8996_mss = { .hexagon_mba_image = "mba.mbn", + .proxy_supply = (struct qcom_mss_reg_res[]) { + { + .supply = "pll", + .uA = 100000, + }, + {} + }, .proxy_clk_names = (char*[]){ "xo", "pnoc", + "qdss", NULL }, .active_clk_names = (char*[]){ "iface", "bus", "mem", - "gpll0_mss_clk", + "gpll0_mss", + "snoc_axi", + "mnoc_axi", NULL }, .need_mem_protection = true, diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index b1e63fcd5fdf4..f280f196d007c 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -258,6 +258,7 @@ static int adsp_probe(struct platform_device *pdev) const struct adsp_data *desc; struct qcom_adsp *adsp; struct rproc *rproc; + const char *fw_name; int ret; desc = of_device_get_match_data(&pdev->dev); @@ -267,8 +268,14 @@ static int adsp_probe(struct platform_device *pdev) if (!qcom_scm_is_available()) return -EPROBE_DEFER; + fw_name = desc->firmware_name; + ret = of_property_read_string(pdev->dev.of_node, "firmware-name", + &fw_name); + if (ret < 0 && ret != -EINVAL) + return ret; + rproc = rproc_alloc(&pdev->dev, pdev->name, &adsp_ops, - desc->firmware_name, sizeof(*adsp)); + fw_name, sizeof(*adsp)); if (!rproc) { dev_err(&pdev->dev, "unable to allocate remoteproc\n"); return -ENOMEM; @@ -304,6 +311,10 @@ static int adsp_probe(struct platform_device *pdev) adsp->sysmon = qcom_add_sysmon_subdev(rproc, desc->sysmon_name, desc->ssctl_id); + if (IS_ERR(adsp->sysmon)) { + ret = PTR_ERR(adsp->sysmon); + goto free_rproc; + } ret = rproc_add(rproc); if (ret) diff --git a/drivers/remoteproc/qcom_sysmon.c b/drivers/remoteproc/qcom_sysmon.c index e976a602b0155..c231314eab667 100644 --- a/drivers/remoteproc/qcom_sysmon.c +++ b/drivers/remoteproc/qcom_sysmon.c @@ -6,8 +6,9 @@ #include <linux/module.h> #include <linux/notifier.h> #include <linux/slab.h> +#include <linux/interrupt.h> #include <linux/io.h> -#include <linux/notifier.h> +#include <linux/of_irq.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/remoteproc/qcom_rproc.h> @@ -25,6 +26,7 @@ struct qcom_sysmon { const char *name; + int shutdown_irq; int ssctl_version; int ssctl_instance; @@ -34,6 +36,8 @@ struct qcom_sysmon { struct rpmsg_endpoint *ept; struct completion comp; + struct completion ind_comp; + struct completion shutdown_comp; struct mutex lock; bool ssr_ack; @@ -137,6 +141,7 @@ static int sysmon_callback(struct rpmsg_device *rpdev, void *data, int count, } #define SSCTL_SHUTDOWN_REQ 0x21 +#define SSCTL_SHUTDOWN_READY_IND 0x21 #define SSCTL_SUBSYS_EVENT_REQ 0x23 #define SSCTL_MAX_MSG_LEN 7 @@ -252,6 +257,29 @@ static struct qmi_elem_info ssctl_subsys_event_resp_ei[] = { {} }; +static struct qmi_elem_info ssctl_shutdown_ind_ei[] = { + {} +}; + +static void sysmon_ind_cb(struct qmi_handle *qmi, struct sockaddr_qrtr *sq, + struct qmi_txn *txn, const void *data) +{ + struct qcom_sysmon *sysmon = container_of(qmi, struct qcom_sysmon, qmi); + + complete(&sysmon->ind_comp); +} + +static struct qmi_msg_handler qmi_indication_handler[] = { + { + .type = QMI_INDICATION, + .msg_id = SSCTL_SHUTDOWN_READY_IND, + .ei = ssctl_shutdown_ind_ei, + .decoded_size = 0, + .fn = sysmon_ind_cb + }, + {} +}; + /** * ssctl_request_shutdown() - request shutdown via SSCTL QMI service * @sysmon: sysmon context @@ -262,6 +290,8 @@ static void ssctl_request_shutdown(struct qcom_sysmon *sysmon) struct qmi_txn txn; int ret; + reinit_completion(&sysmon->ind_comp); + reinit_completion(&sysmon->shutdown_comp); ret = qmi_txn_init(&sysmon->qmi, &txn, ssctl_shutdown_resp_ei, &resp); if (ret < 0) { dev_err(sysmon->dev, "failed to allocate QMI txn\n"); @@ -283,6 +313,17 @@ static void ssctl_request_shutdown(struct qcom_sysmon *sysmon) dev_err(sysmon->dev, "shutdown request failed\n"); else dev_dbg(sysmon->dev, "shutdown request completed\n"); + + if (sysmon->shutdown_irq > 0) { + ret = wait_for_completion_timeout(&sysmon->shutdown_comp, + 10 * HZ); + if (!ret) { + ret = try_wait_for_completion(&sysmon->ind_comp); + if (!ret) + dev_err(sysmon->dev, + "timeout waiting for shutdown ack\n"); + } + } } /** @@ -432,6 +473,15 @@ static int sysmon_notify(struct notifier_block *nb, unsigned long event, return NOTIFY_DONE; } +static irqreturn_t sysmon_shutdown_interrupt(int irq, void *data) +{ + struct qcom_sysmon *sysmon = data; + + complete(&sysmon->shutdown_comp); + + return IRQ_HANDLED; +} + /** * qcom_add_sysmon_subdev() - create a sysmon subdev for the given remoteproc * @rproc: rproc context to associate the subdev with @@ -449,7 +499,7 @@ struct qcom_sysmon *qcom_add_sysmon_subdev(struct rproc *rproc, sysmon = kzalloc(sizeof(*sysmon), GFP_KERNEL); if (!sysmon) - return NULL; + return ERR_PTR(-ENOMEM); sysmon->dev = rproc->dev.parent; sysmon->rproc = rproc; @@ -458,13 +508,37 @@ struct qcom_sysmon *qcom_add_sysmon_subdev(struct rproc *rproc, sysmon->ssctl_instance = ssctl_instance; init_completion(&sysmon->comp); + init_completion(&sysmon->ind_comp); + init_completion(&sysmon->shutdown_comp); mutex_init(&sysmon->lock); - ret = qmi_handle_init(&sysmon->qmi, SSCTL_MAX_MSG_LEN, &ssctl_ops, NULL); + sysmon->shutdown_irq = of_irq_get_byname(sysmon->dev->of_node, + "shutdown-ack"); + if (sysmon->shutdown_irq < 0) { + if (sysmon->shutdown_irq != -ENODATA) { + dev_err(sysmon->dev, + "failed to retrieve shutdown-ack IRQ\n"); + return ERR_PTR(sysmon->shutdown_irq); + } + } else { + ret = devm_request_threaded_irq(sysmon->dev, + sysmon->shutdown_irq, + NULL, sysmon_shutdown_interrupt, + IRQF_TRIGGER_RISING | IRQF_ONESHOT, + "q6v5 shutdown-ack", sysmon); + if (ret) { + dev_err(sysmon->dev, + "failed to acquire shutdown-ack IRQ\n"); + return ERR_PTR(ret); + } + } + + ret = qmi_handle_init(&sysmon->qmi, SSCTL_MAX_MSG_LEN, &ssctl_ops, + qmi_indication_handler); if (ret < 0) { dev_err(sysmon->dev, "failed to initialize qmi handle\n"); kfree(sysmon); - return NULL; + return ERR_PTR(ret); } qmi_add_lookup(&sysmon->qmi, 43, 0, 0); diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c index b0e07e9f42d56..adcce523971e6 100644 --- a/drivers/remoteproc/qcom_wcnss.c +++ b/drivers/remoteproc/qcom_wcnss.c @@ -553,6 +553,10 @@ static int wcnss_probe(struct platform_device *pdev) qcom_add_smd_subdev(rproc, &wcnss->smd_subdev); wcnss->sysmon = qcom_add_sysmon_subdev(rproc, "wcnss", WCNSS_SSCTL_ID); + if (IS_ERR(wcnss->sysmon)) { + ret = PTR_ERR(wcnss->sysmon); + goto free_rproc; + } ret = rproc_add(rproc); if (ret) @@ -622,5 +626,5 @@ static void __exit wcnss_exit(void) } module_exit(wcnss_exit); -MODULE_DESCRIPTION("Qualcomm Peripherial Image Loader for Wireless Subsystem"); +MODULE_DESCRIPTION("Qualcomm Peripheral Image Loader for Wireless Subsystem"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 54ec38fc5dca6..48feebd6d0a2d 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -39,12 +39,16 @@ #include <linux/idr.h> #include <linux/elf.h> #include <linux/crc32.h> +#include <linux/of_reserved_mem.h> #include <linux/virtio_ids.h> #include <linux/virtio_ring.h> #include <asm/byteorder.h> +#include <linux/platform_device.h> #include "remoteproc_internal.h" +#define HIGH_BITS_MASK 0xFFFFFFFF00000000ULL + static DEFINE_MUTEX(rproc_list_mutex); static LIST_HEAD(rproc_list); @@ -145,7 +149,7 @@ static void rproc_disable_iommu(struct rproc *rproc) iommu_domain_free(domain); } -static phys_addr_t rproc_va_to_pa(void *cpu_addr) +phys_addr_t rproc_va_to_pa(void *cpu_addr) { /* * Return physical address according to virtual address location @@ -160,6 +164,7 @@ static phys_addr_t rproc_va_to_pa(void *cpu_addr) WARN_ON(!virt_addr_valid(cpu_addr)); return virt_to_phys(cpu_addr); } +EXPORT_SYMBOL(rproc_va_to_pa); /** * rproc_da_to_va() - lookup the kernel virtual address for a remoteproc address @@ -204,6 +209,10 @@ void *rproc_da_to_va(struct rproc *rproc, u64 da, int len) list_for_each_entry(carveout, &rproc->carveouts, node) { int offset = da - carveout->da; + /* Verify that carveout is allocated */ + if (!carveout->va) + continue; + /* try next carveout if da is too small */ if (offset < 0) continue; @@ -272,25 +281,27 @@ rproc_find_carveout_by_name(struct rproc *rproc, const char *name, ...) * @len: associated area size * * This function is a helper function to verify requested device area (couple - * da, len) is part of specified carevout. + * da, len) is part of specified carveout. + * If da is not set (defined as FW_RSC_ADDR_ANY), only requested length is + * checked. * - * Return: 0 if carveout match request else -ENOMEM + * Return: 0 if carveout matches request else error */ -int rproc_check_carveout_da(struct rproc *rproc, struct rproc_mem_entry *mem, - u32 da, u32 len) +static int rproc_check_carveout_da(struct rproc *rproc, + struct rproc_mem_entry *mem, u32 da, u32 len) { struct device *dev = &rproc->dev; - int delta = 0; + int delta; /* Check requested resource length */ if (len > mem->len) { dev_err(dev, "Registered carveout doesn't fit len request\n"); - return -ENOMEM; + return -EINVAL; } if (da != FW_RSC_ADDR_ANY && mem->da == FW_RSC_ADDR_ANY) { - /* Update existing carveout da */ - mem->da = da; + /* Address doesn't match registered carveout configuration */ + return -EINVAL; } else if (da != FW_RSC_ADDR_ANY && mem->da != FW_RSC_ADDR_ANY) { delta = da - mem->da; @@ -298,13 +309,13 @@ int rproc_check_carveout_da(struct rproc *rproc, struct rproc_mem_entry *mem, if (delta < 0) { dev_err(dev, "Registered carveout doesn't fit da request\n"); - return -ENOMEM; + return -EINVAL; } if (delta + len > mem->len) { dev_err(dev, "Registered carveout doesn't fit len request\n"); - return -ENOMEM; + return -EINVAL; } } @@ -418,8 +429,25 @@ static int rproc_vdev_do_start(struct rproc_subdev *subdev) static void rproc_vdev_do_stop(struct rproc_subdev *subdev, bool crashed) { struct rproc_vdev *rvdev = container_of(subdev, struct rproc_vdev, subdev); + int ret; - rproc_remove_virtio_dev(rvdev); + ret = device_for_each_child(&rvdev->dev, NULL, rproc_remove_virtio_dev); + if (ret) + dev_warn(&rvdev->dev, "can't remove vdev child device: %d\n", ret); +} + +/** + * rproc_rvdev_release() - release the existence of a rvdev + * + * @dev: the subdevice's dev + */ +static void rproc_rvdev_release(struct device *dev) +{ + struct rproc_vdev *rvdev = container_of(dev, struct rproc_vdev, dev); + + of_reserved_mem_device_release(dev); + + kfree(rvdev); } /** @@ -455,6 +483,7 @@ static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc, struct device *dev = &rproc->dev; struct rproc_vdev *rvdev; int i, ret; + char name[16]; /* make sure resource isn't truncated */ if (sizeof(*rsc) + rsc->num_of_vrings * sizeof(struct fw_rsc_vdev_vring) @@ -488,6 +517,29 @@ static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc, rvdev->rproc = rproc; rvdev->index = rproc->nb_vdev++; + /* Initialise vdev subdevice */ + snprintf(name, sizeof(name), "vdev%dbuffer", rvdev->index); + rvdev->dev.parent = rproc->dev.parent; + rvdev->dev.release = rproc_rvdev_release; + dev_set_name(&rvdev->dev, "%s#%s", dev_name(rvdev->dev.parent), name); + dev_set_drvdata(&rvdev->dev, rvdev); + + ret = device_register(&rvdev->dev); + if (ret) { + put_device(&rvdev->dev); + return ret; + } + /* Make device dma capable by inheriting from parent's capabilities */ + set_dma_ops(&rvdev->dev, get_dma_ops(rproc->dev.parent)); + + ret = dma_coerce_mask_and_coherent(&rvdev->dev, + dma_get_mask(rproc->dev.parent)); + if (ret) { + dev_warn(dev, + "Failed to set DMA mask %llx. Trying to continue... %x\n", + dma_get_mask(rproc->dev.parent), ret); + } + /* parse the vrings */ for (i = 0; i < rsc->num_of_vrings; i++) { ret = rproc_parse_vring(rvdev, rsc, i); @@ -518,7 +570,7 @@ unwind_vring_allocations: for (i--; i >= 0; i--) rproc_free_vring(&rvdev->vring[i]); free_rvdev: - kfree(rvdev); + device_unregister(&rvdev->dev); return ret; } @@ -536,7 +588,7 @@ void rproc_vdev_release(struct kref *ref) rproc_remove_subdev(rproc, &rvdev->subdev); list_del(&rvdev->node); - kfree(rvdev); + device_unregister(&rvdev->dev); } /** @@ -558,9 +610,8 @@ void rproc_vdev_release(struct kref *ref) static int rproc_handle_trace(struct rproc *rproc, struct fw_rsc_trace *rsc, int offset, int avail) { - struct rproc_mem_entry *trace; + struct rproc_debug_trace *trace; struct device *dev = &rproc->dev; - void *ptr; char name[15]; if (sizeof(*rsc) > avail) { @@ -574,28 +625,23 @@ static int rproc_handle_trace(struct rproc *rproc, struct fw_rsc_trace *rsc, return -EINVAL; } - /* what's the kernel address of this resource ? */ - ptr = rproc_da_to_va(rproc, rsc->da, rsc->len); - if (!ptr) { - dev_err(dev, "erroneous trace resource entry\n"); - return -EINVAL; - } - trace = kzalloc(sizeof(*trace), GFP_KERNEL); if (!trace) return -ENOMEM; /* set the trace buffer dma properties */ - trace->len = rsc->len; - trace->va = ptr; + trace->trace_mem.len = rsc->len; + trace->trace_mem.da = rsc->da; + + /* set pointer on rproc device */ + trace->rproc = rproc; /* make sure snprintf always null terminates, even if truncating */ snprintf(name, sizeof(name), "trace%d", rproc->num_traces); /* create the debugfs entry */ - trace->priv = rproc_create_trace_file(name, rproc, trace); - if (!trace->priv) { - trace->va = NULL; + trace->tfile = rproc_create_trace_file(name, rproc, trace); + if (!trace->tfile) { kfree(trace); return -EINVAL; } @@ -604,8 +650,8 @@ static int rproc_handle_trace(struct rproc *rproc, struct fw_rsc_trace *rsc, rproc->num_traces++; - dev_dbg(dev, "%s added: va %pK, da 0x%x, len 0x%x\n", - name, ptr, rsc->da, rsc->len); + dev_dbg(dev, "%s added: da 0x%x, len 0x%x\n", + name, rsc->da, rsc->len); return 0; } @@ -715,6 +761,18 @@ static int rproc_alloc_carveout(struct rproc *rproc, dev_dbg(dev, "carveout va %pK, dma %pad, len 0x%x\n", va, &dma, mem->len); + if (mem->da != FW_RSC_ADDR_ANY && !rproc->domain) { + /* + * Check requested da is equal to dma address + * and print a warn message in case of missalignment. + * Don't stop rproc_start sequence as coprocessor may + * build pa to da translation on its side. + */ + if (mem->da != (u32)dma) + dev_warn(dev->parent, + "Allocated carveout doesn't fit device address request\n"); + } + /* * Ok, this is non-standard. * @@ -732,15 +790,7 @@ static int rproc_alloc_carveout(struct rproc *rproc, * to use the iommu-based DMA API: we expect 'dma' to contain the * physical address in this case. */ - - if (mem->da != FW_RSC_ADDR_ANY) { - if (!rproc->domain) { - dev_err(dev->parent, - "Bad carveout rsc configuration\n"); - ret = -ENOMEM; - goto dma_free; - } - + if (mem->da != FW_RSC_ADDR_ANY && rproc->domain) { mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); if (!mapping) { ret = -ENOMEM; @@ -767,11 +817,17 @@ static int rproc_alloc_carveout(struct rproc *rproc, dev_dbg(dev, "carveout mapped 0x%x to %pad\n", mem->da, &dma); - } else { + } + + if (mem->da == FW_RSC_ADDR_ANY) { + /* Update device address as undefined by requester */ + if ((u64)dma & HIGH_BITS_MASK) + dev_warn(dev, "DMA address cast in 32bit to fit resource table format\n"); + mem->da = (u32)dma; } - mem->dma = (u32)dma; + mem->dma = dma; mem->va = va; return 0; @@ -900,7 +956,8 @@ EXPORT_SYMBOL(rproc_add_carveout); * @dma: dma address * @len: memory carveout length * @da: device address - * @release: memory carveout function + * @alloc: memory carveout allocation function + * @release: memory carveout release function * @name: carveout name * * This function allocates a rproc_mem_entry struct and fill it with parameters @@ -1110,6 +1167,7 @@ static int rproc_alloc_registered_carveouts(struct rproc *rproc) struct rproc_mem_entry *entry, *tmp; struct fw_rsc_carveout *rsc; struct device *dev = &rproc->dev; + u64 pa; int ret; list_for_each_entry_safe(entry, tmp, &rproc->carveouts, node) { @@ -1146,10 +1204,15 @@ static int rproc_alloc_registered_carveouts(struct rproc *rproc) /* Use va if defined else dma to generate pa */ if (entry->va) - rsc->pa = (u32)rproc_va_to_pa(entry->va); + pa = (u64)rproc_va_to_pa(entry->va); else - rsc->pa = (u32)entry->dma; + pa = (u64)entry->dma; + + if (((u64)pa) & HIGH_BITS_MASK) + dev_warn(dev, + "Physical address cast in 32bit to fit resource table format\n"); + rsc->pa = (u32)pa; rsc->da = entry->da; rsc->len = entry->len; } @@ -1182,15 +1245,16 @@ static void rproc_coredump_cleanup(struct rproc *rproc) static void rproc_resource_cleanup(struct rproc *rproc) { struct rproc_mem_entry *entry, *tmp; + struct rproc_debug_trace *trace, *ttmp; struct rproc_vdev *rvdev, *rvtmp; struct device *dev = &rproc->dev; /* clean up debugfs trace entries */ - list_for_each_entry_safe(entry, tmp, &rproc->traces, node) { - rproc_remove_trace_file(entry->priv); + list_for_each_entry_safe(trace, ttmp, &rproc->traces, node) { + rproc_remove_trace_file(trace->tfile); rproc->num_traces--; - list_del(&entry->node); - kfree(entry); + list_del(&trace->node); + kfree(trace); } /* clean up iommu mapping entries */ diff --git a/drivers/remoteproc/remoteproc_debugfs.c b/drivers/remoteproc/remoteproc_debugfs.c index e90135c64af07..6da934b8dc4be 100644 --- a/drivers/remoteproc/remoteproc_debugfs.c +++ b/drivers/remoteproc/remoteproc_debugfs.c @@ -47,10 +47,23 @@ static struct dentry *rproc_dbg; static ssize_t rproc_trace_read(struct file *filp, char __user *userbuf, size_t count, loff_t *ppos) { - struct rproc_mem_entry *trace = filp->private_data; - int len = strnlen(trace->va, trace->len); + struct rproc_debug_trace *data = filp->private_data; + struct rproc_mem_entry *trace = &data->trace_mem; + void *va; + char buf[100]; + int len; + + va = rproc_da_to_va(data->rproc, trace->da, trace->len); - return simple_read_from_buffer(userbuf, count, ppos, trace->va, len); + if (!va) { + len = scnprintf(buf, sizeof(buf), "Trace %s not available\n", + trace->name); + va = buf; + } else { + len = strnlen(va, trace->len); + } + + return simple_read_from_buffer(userbuf, count, ppos, va, len); } static const struct file_operations trace_rproc_ops = { @@ -155,6 +168,30 @@ static const struct file_operations rproc_recovery_ops = { .llseek = generic_file_llseek, }; +/* expose the crash trigger via debugfs */ +static ssize_t +rproc_crash_write(struct file *filp, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct rproc *rproc = filp->private_data; + unsigned int type; + int ret; + + ret = kstrtouint_from_user(user_buf, count, 0, &type); + if (ret < 0) + return ret; + + rproc_report_crash(rproc, type); + + return count; +} + +static const struct file_operations rproc_crash_ops = { + .write = rproc_crash_write, + .open = simple_open, + .llseek = generic_file_llseek, +}; + /* Expose resource table content via debugfs */ static int rproc_rsc_table_show(struct seq_file *seq, void *p) { @@ -288,7 +325,7 @@ void rproc_remove_trace_file(struct dentry *tfile) } struct dentry *rproc_create_trace_file(const char *name, struct rproc *rproc, - struct rproc_mem_entry *trace) + struct rproc_debug_trace *trace) { struct dentry *tfile; @@ -325,6 +362,8 @@ void rproc_create_debug_dir(struct rproc *rproc) rproc, &rproc_name_ops); debugfs_create_file("recovery", 0400, rproc->dbg_dir, rproc, &rproc_recovery_ops); + debugfs_create_file("crash", 0200, rproc->dbg_dir, + rproc, &rproc_crash_ops); debugfs_create_file("resource_table", 0400, rproc->dbg_dir, rproc, &rproc_rsc_table_ops); debugfs_create_file("carveout_memories", 0400, rproc->dbg_dir, diff --git a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h index f6cad243d7cac..45ff76a06c72b 100644 --- a/drivers/remoteproc/remoteproc_internal.h +++ b/drivers/remoteproc/remoteproc_internal.h @@ -25,6 +25,13 @@ struct rproc; +struct rproc_debug_trace { + struct rproc *rproc; + struct dentry *tfile; + struct list_head node; + struct rproc_mem_entry trace_mem; +}; + /* from remoteproc_core.c */ void rproc_release(struct kref *kref); irqreturn_t rproc_vq_interrupt(struct rproc *rproc, int vq_id); @@ -32,12 +39,12 @@ void rproc_vdev_release(struct kref *ref); /* from remoteproc_virtio.c */ int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id); -void rproc_remove_virtio_dev(struct rproc_vdev *rvdev); +int rproc_remove_virtio_dev(struct device *dev, void *data); /* from remoteproc_debugfs.c */ void rproc_remove_trace_file(struct dentry *tfile); struct dentry *rproc_create_trace_file(const char *name, struct rproc *rproc, - struct rproc_mem_entry *trace); + struct rproc_debug_trace *trace); void rproc_delete_debug_dir(struct rproc *rproc); void rproc_create_debug_dir(struct rproc *rproc); void rproc_init_debugfs(void); @@ -52,6 +59,7 @@ void rproc_free_vring(struct rproc_vring *rvring); int rproc_alloc_vring(struct rproc_vdev *rvdev, int i); void *rproc_da_to_va(struct rproc *rproc, u64 da, int len); +phys_addr_t rproc_va_to_pa(void *cpu_addr); int rproc_trigger_recovery(struct rproc *rproc); int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw); diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index 2d7cd344f3bff..44774de6f17b0 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -17,7 +17,9 @@ * GNU General Public License for more details. */ +#include <linux/dma-mapping.h> #include <linux/export.h> +#include <linux/of_reserved_mem.h> #include <linux/remoteproc.h> #include <linux/virtio.h> #include <linux/virtio_config.h> @@ -316,6 +318,8 @@ static void rproc_virtio_dev_release(struct device *dev) struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); struct rproc *rproc = vdev_to_rproc(vdev); + kfree(vdev); + kref_put(&rvdev->refcount, rproc_vdev_release); put_device(&rproc->dev); @@ -333,10 +337,53 @@ static void rproc_virtio_dev_release(struct device *dev) int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id) { struct rproc *rproc = rvdev->rproc; - struct device *dev = &rproc->dev; - struct virtio_device *vdev = &rvdev->vdev; + struct device *dev = &rvdev->dev; + struct virtio_device *vdev; + struct rproc_mem_entry *mem; int ret; + /* Try to find dedicated vdev buffer carveout */ + mem = rproc_find_carveout_by_name(rproc, "vdev%dbuffer", rvdev->index); + if (mem) { + phys_addr_t pa; + + if (mem->of_resm_idx != -1) { + struct device_node *np = rproc->dev.parent->of_node; + + /* Associate reserved memory to vdev device */ + ret = of_reserved_mem_device_init_by_idx(dev, np, + mem->of_resm_idx); + if (ret) { + dev_err(dev, "Can't associate reserved memory\n"); + goto out; + } + } else { + if (mem->va) { + dev_warn(dev, "vdev %d buffer already mapped\n", + rvdev->index); + pa = rproc_va_to_pa(mem->va); + } else { + /* Use dma address as carveout no memmapped yet */ + pa = (phys_addr_t)mem->dma; + } + + /* Associate vdev buffer memory pool to vdev subdev */ + ret = dma_declare_coherent_memory(dev, pa, + mem->da, + mem->len); + if (ret < 0) { + dev_err(dev, "Failed to associate buffer\n"); + goto out; + } + } + } + + /* Allocate virtio device */ + vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); + if (!vdev) { + ret = -ENOMEM; + goto out; + } vdev->id.device = id, vdev->config = &rproc_virtio_config_ops, vdev->dev.parent = dev; @@ -370,11 +417,15 @@ out: /** * rproc_remove_virtio_dev() - remove an rproc-induced virtio device - * @rvdev: the remote vdev + * @dev: the virtio device + * @data: must be null * * This function unregisters an existing virtio device. */ -void rproc_remove_virtio_dev(struct rproc_vdev *rvdev) +int rproc_remove_virtio_dev(struct device *dev, void *data) { - unregister_virtio_device(&rvdev->vdev); + struct virtio_device *vdev = dev_to_virtio(dev); + + unregister_virtio_device(vdev); + return 0; } diff --git a/drivers/remoteproc/st_remoteproc.c b/drivers/remoteproc/st_remoteproc.c index aacef0ea3b902..51049d17b1e5c 100644 --- a/drivers/remoteproc/st_remoteproc.c +++ b/drivers/remoteproc/st_remoteproc.c @@ -19,6 +19,7 @@ #include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/of_address.h> #include <linux/of_device.h> #include <linux/of_reserved_mem.h> #include <linux/platform_device.h> @@ -91,6 +92,77 @@ static void st_rproc_kick(struct rproc *rproc, int vqid) dev_err(dev, "failed to send message via mbox: %d\n", ret); } +static int st_rproc_mem_alloc(struct rproc *rproc, + struct rproc_mem_entry *mem) +{ + struct device *dev = rproc->dev.parent; + void *va; + + va = ioremap_wc(mem->dma, mem->len); + if (!va) { + dev_err(dev, "Unable to map memory region: %pa+%zx\n", + &mem->dma, mem->len); + return -ENOMEM; + } + + /* Update memory entry va */ + mem->va = va; + + return 0; +} + +static int st_rproc_mem_release(struct rproc *rproc, + struct rproc_mem_entry *mem) +{ + iounmap(mem->va); + + return 0; +} + +static int st_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw) +{ + struct device *dev = rproc->dev.parent; + struct device_node *np = dev->of_node; + struct rproc_mem_entry *mem; + struct reserved_mem *rmem; + struct of_phandle_iterator it; + int index = 0; + + of_phandle_iterator_init(&it, np, "memory-region", NULL, 0); + while (of_phandle_iterator_next(&it) == 0) { + rmem = of_reserved_mem_lookup(it.node); + if (!rmem) { + dev_err(dev, "unable to acquire memory-region\n"); + return -EINVAL; + } + + /* No need to map vdev buffer */ + if (strcmp(it.node->name, "vdev0buffer")) { + /* Register memory region */ + mem = rproc_mem_entry_init(dev, NULL, + (dma_addr_t)rmem->base, + rmem->size, rmem->base, + st_rproc_mem_alloc, + st_rproc_mem_release, + it.node->name); + } else { + /* Register reserved memory for vdev buffer allocation */ + mem = rproc_of_resm_mem_entry_init(dev, index, + rmem->size, + rmem->base, + it.node->name); + } + + if (!mem) + return -ENOMEM; + + rproc_add_carveout(rproc, mem); + index++; + } + + return rproc_elf_load_rsc_table(rproc, fw); +} + static int st_rproc_start(struct rproc *rproc) { struct st_rproc *ddata = rproc->priv; @@ -158,9 +230,14 @@ static int st_rproc_stop(struct rproc *rproc) } static const struct rproc_ops st_rproc_ops = { - .kick = st_rproc_kick, - .start = st_rproc_start, - .stop = st_rproc_stop, + .kick = st_rproc_kick, + .start = st_rproc_start, + .stop = st_rproc_stop, + .parse_fw = st_rproc_parse_fw, + .load = rproc_elf_load_segments, + .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table, + .sanity_check = rproc_elf_sanity_check, + .get_boot_addr = rproc_elf_get_boot_addr, }; /* @@ -254,12 +331,6 @@ static int st_rproc_parse_dt(struct platform_device *pdev) return -EINVAL; } - err = of_reserved_mem_device_init(dev); - if (err) { - dev_err(dev, "Failed to obtain shared memory\n"); - return err; - } - err = clk_prepare(ddata->clk); if (err) dev_err(dev, "failed to get clock\n"); @@ -387,8 +458,6 @@ static int st_rproc_remove(struct platform_device *pdev) clk_disable_unprepare(ddata->clk); - of_reserved_mem_device_release(&pdev->dev); - for (i = 0; i < ST_RPROC_MAX_VRING * MBOX_MAX; i++) mbox_free_channel(ddata->mbox_chan[i]); diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c index 664f957012cde..5d3685bd76a2d 100644 --- a/drivers/rpmsg/virtio_rpmsg_bus.c +++ b/drivers/rpmsg/virtio_rpmsg_bus.c @@ -11,21 +11,21 @@ #define pr_fmt(fmt) "%s: " fmt, __func__ +#include <linux/dma-mapping.h> +#include <linux/idr.h> +#include <linux/jiffies.h> #include <linux/kernel.h> #include <linux/module.h> -#include <linux/virtio.h> -#include <linux/virtio_ids.h> -#include <linux/virtio_config.h> +#include <linux/mutex.h> +#include <linux/of_device.h> +#include <linux/rpmsg.h> #include <linux/scatterlist.h> -#include <linux/dma-mapping.h> #include <linux/slab.h> -#include <linux/idr.h> -#include <linux/jiffies.h> #include <linux/sched.h> +#include <linux/virtio.h> +#include <linux/virtio_ids.h> +#include <linux/virtio_config.h> #include <linux/wait.h> -#include <linux/rpmsg.h> -#include <linux/mutex.h> -#include <linux/of_device.h> #include "rpmsg_internal.h" @@ -912,7 +912,7 @@ static int rpmsg_probe(struct virtio_device *vdev) total_buf_space = vrp->num_bufs * vrp->buf_size; /* allocate coherent memory for the buffers */ - bufs_va = dma_alloc_coherent(vdev->dev.parent->parent, + bufs_va = dma_alloc_coherent(vdev->dev.parent, total_buf_space, &vrp->bufs_dma, GFP_KERNEL); if (!bufs_va) { @@ -980,7 +980,7 @@ static int rpmsg_probe(struct virtio_device *vdev) return 0; free_coherent: - dma_free_coherent(vdev->dev.parent->parent, total_buf_space, + dma_free_coherent(vdev->dev.parent, total_buf_space, bufs_va, vrp->bufs_dma); vqs_del: vdev->config->del_vqs(vrp->vdev); @@ -1015,7 +1015,7 @@ static void rpmsg_remove(struct virtio_device *vdev) vdev->config->del_vqs(vrp->vdev); - dma_free_coherent(vdev->dev.parent->parent, total_buf_space, + dma_free_coherent(vdev->dev.parent, total_buf_space, vrp->rbufs, vrp->bufs_dma); kfree(vrp); diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index a0baee25134c0..4159c63a5fd2b 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -1382,3 +1382,40 @@ int chsc_pnso_brinfo(struct subchannel_id schid, return chsc_error_from_response(brinfo_area->response.code); } EXPORT_SYMBOL_GPL(chsc_pnso_brinfo); + +int chsc_sgib(u32 origin) +{ + struct { + struct chsc_header request; + u16 op; + u8 reserved01[2]; + u8 reserved02:4; + u8 fmt:4; + u8 reserved03[7]; + /* operation data area begin */ + u8 reserved04[4]; + u32 gib_origin; + u8 reserved05[10]; + u8 aix; + u8 reserved06[4029]; + struct chsc_header response; + u8 reserved07[4]; + } *sgib_area; + int ret; + + spin_lock_irq(&chsc_page_lock); + memset(chsc_page, 0, PAGE_SIZE); + sgib_area = chsc_page; + sgib_area->request.length = 0x0fe0; + sgib_area->request.code = 0x0021; + sgib_area->op = 0x1; + sgib_area->gib_origin = origin; + + ret = chsc(sgib_area); + if (ret == 0) + ret = chsc_error_from_response(sgib_area->response.code); + spin_unlock_irq(&chsc_page_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(chsc_sgib); diff --git a/drivers/s390/cio/chsc.h b/drivers/s390/cio/chsc.h index 78aba8d94eec3..e57d68e325a37 100644 --- a/drivers/s390/cio/chsc.h +++ b/drivers/s390/cio/chsc.h @@ -164,6 +164,7 @@ int chsc_get_channel_measurement_chars(struct channel_path *chp); int chsc_ssqd(struct subchannel_id schid, struct chsc_ssqd_area *ssqd); int chsc_sadc(struct subchannel_id schid, struct chsc_scssc_area *scssc, u64 summary_indicator_addr, u64 subchannel_indicator_addr); +int chsc_sgib(u32 origin); int chsc_error_from_response(int response); int chsc_siosl(struct subchannel_id schid); diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c index a45f81ec80cec..8e28a505f7e8c 100644 --- a/drivers/scsi/aacraid/linit.c +++ b/drivers/scsi/aacraid/linit.c @@ -413,13 +413,16 @@ static int aac_slave_configure(struct scsi_device *sdev) if (chn < AAC_MAX_BUSES && tid < AAC_MAX_TARGETS && aac->sa_firmware) { devtype = aac->hba_map[chn][tid].devtype; - if (devtype == AAC_DEVTYPE_NATIVE_RAW) + if (devtype == AAC_DEVTYPE_NATIVE_RAW) { depth = aac->hba_map[chn][tid].qd_limit; - else if (devtype == AAC_DEVTYPE_ARC_RAW) + set_timeout = 1; + goto common_config; + } + if (devtype == AAC_DEVTYPE_ARC_RAW) { set_qd_dev_type = true; - - set_timeout = 1; - goto common_config; + set_timeout = 1; + goto common_config; + } } if (aac->jbod && (sdev->type == TYPE_DISK)) diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h index 6c87bd34509af..9bfa9f12d81e8 100644 --- a/drivers/scsi/hisi_sas/hisi_sas.h +++ b/drivers/scsi/hisi_sas/hisi_sas.h @@ -18,6 +18,7 @@ #include <linux/dmapool.h> #include <linux/iopoll.h> #include <linux/lcm.h> +#include <linux/libata.h> #include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/of_address.h> @@ -94,6 +95,11 @@ enum { PORT_TYPE_SATA = (1U << 0), }; +enum dev_status { + HISI_SAS_DEV_INIT, + HISI_SAS_DEV_NORMAL, +}; + enum { HISI_SAS_INT_ABT_CMD = 0, HISI_SAS_INT_ABT_DEV = 1, @@ -161,6 +167,7 @@ struct hisi_sas_phy { u8 in_reset; u8 reserved[2]; u32 phy_type; + u32 code_violation_err_count; enum sas_linkrate minimum_linkrate; enum sas_linkrate maximum_linkrate; }; @@ -194,6 +201,7 @@ struct hisi_sas_device { struct hisi_sas_dq *dq; struct list_head list; enum sas_device_type dev_type; + enum dev_status dev_status; int device_id; int sata_idx; spinlock_t lock; /* For protecting slots */ diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c index 13ca5a0bdf6be..3c3cf89f713fb 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_main.c +++ b/drivers/scsi/hisi_sas/hisi_sas_main.c @@ -10,6 +10,7 @@ */ #include "hisi_sas.h" +#include "../libsas/sas_internal.h" #define DRV_NAME "hisi_sas" #define DEV_IS_GONE(dev) \ @@ -707,6 +708,7 @@ static struct hisi_sas_device *hisi_sas_alloc_dev(struct domain_device *device) hisi_hba->devices[i].device_id = i; sas_dev = &hisi_hba->devices[i]; + sas_dev->dev_status = HISI_SAS_DEV_INIT; sas_dev->dev_type = device->dev_type; sas_dev->hisi_hba = hisi_hba; sas_dev->sas_device = device; @@ -731,6 +733,8 @@ static int hisi_sas_init_device(struct domain_device *device) struct hisi_sas_tmf_task tmf_task; int retry = HISI_SAS_SRST_ATA_DISK_CNT; struct hisi_hba *hisi_hba = dev_to_hisi_hba(device); + struct device *dev = hisi_hba->dev; + struct sas_phy *local_phy; switch (device->dev_type) { case SAS_END_DEVICE: @@ -746,6 +750,31 @@ static int hisi_sas_init_device(struct domain_device *device) case SAS_SATA_PM: case SAS_SATA_PM_PORT: case SAS_SATA_PENDING: + /* + * send HARD RESET to clear previous affiliation of + * STP target port + */ + local_phy = sas_get_local_phy(device); + if (!scsi_is_sas_phy_local(local_phy)) { + unsigned long deadline = ata_deadline(jiffies, 20000); + struct sata_device *sata_dev = &device->sata_dev; + struct ata_host *ata_host = sata_dev->ata_host; + struct ata_port_operations *ops = ata_host->ops; + struct ata_port *ap = sata_dev->ap; + struct ata_link *link; + unsigned int classes; + + ata_for_each_link(link, ap, EDGE) + rc = ops->hardreset(link, &classes, + deadline); + } + sas_put_local_phy(local_phy); + if (rc) { + dev_warn(dev, "SATA disk hardreset fail: 0x%x\n", + rc); + return rc; + } + while (retry-- > 0) { rc = hisi_sas_softreset_ata_disk(device); if (!rc) @@ -808,6 +837,7 @@ static int hisi_sas_dev_found(struct domain_device *device) rc = hisi_sas_init_device(device); if (rc) goto err_out; + sas_dev->dev_status = HISI_SAS_DEV_NORMAL; return 0; err_out: @@ -980,7 +1010,8 @@ static void hisi_sas_do_release_task(struct hisi_hba *hisi_hba, struct sas_task spin_lock_irqsave(&task->task_state_lock, flags); task->task_state_flags &= ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR); - task->task_state_flags |= SAS_TASK_STATE_DONE; + if (!slot->is_internal && task->task_proto != SAS_PROTOCOL_SMP) + task->task_state_flags |= SAS_TASK_STATE_DONE; spin_unlock_irqrestore(&task->task_state_lock, flags); } @@ -1713,20 +1744,23 @@ static int hisi_sas_clear_aca(struct domain_device *device, u8 *lun) static int hisi_sas_debug_I_T_nexus_reset(struct domain_device *device) { struct sas_phy *local_phy = sas_get_local_phy(device); - int rc, reset_type = (device->dev_type == SAS_SATA_DEV || - (device->tproto & SAS_PROTOCOL_STP)) ? 0 : 1; + struct hisi_sas_device *sas_dev = device->lldd_dev; struct hisi_hba *hisi_hba = dev_to_hisi_hba(device); struct sas_ha_struct *sas_ha = &hisi_hba->sha; struct asd_sas_phy *sas_phy = sas_ha->sas_phy[local_phy->number]; struct hisi_sas_phy *phy = container_of(sas_phy, struct hisi_sas_phy, sas_phy); DECLARE_COMPLETION_ONSTACK(phyreset); + int rc, reset_type; if (scsi_is_sas_phy_local(local_phy)) { phy->in_reset = 1; phy->reset_completion = &phyreset; } + reset_type = (sas_dev->dev_status == HISI_SAS_DEV_INIT || + !dev_is_sata(device)) ? 1 : 0; + rc = sas_phy_reset(local_phy, reset_type); sas_put_local_phy(local_phy); @@ -1742,8 +1776,13 @@ static int hisi_sas_debug_I_T_nexus_reset(struct domain_device *device) /* report PHY down if timed out */ if (!ret) hisi_sas_phy_down(hisi_hba, sas_phy->id, 0); - } else + } else if (sas_dev->dev_status != HISI_SAS_DEV_INIT) { + /* + * If in init state, we rely on caller to wait for link to be + * ready; otherwise, delay. + */ msleep(2000); + } return rc; } @@ -2125,9 +2164,18 @@ static int hisi_sas_write_gpio(struct sas_ha_struct *sha, u8 reg_type, static void hisi_sas_phy_disconnected(struct hisi_sas_phy *phy) { + struct asd_sas_phy *sas_phy = &phy->sas_phy; + struct sas_phy *sphy = sas_phy->phy; + struct sas_phy_data *d = sphy->hostdata; + phy->phy_attached = 0; phy->phy_type = 0; phy->port = NULL; + + if (d->enable) + sphy->negotiated_linkrate = SAS_LINK_RATE_UNKNOWN; + else + sphy->negotiated_linkrate = SAS_PHY_DISABLED; } void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy) @@ -2253,6 +2301,7 @@ int hisi_sas_alloc(struct hisi_hba *hisi_hba) for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) { hisi_hba->devices[i].dev_type = SAS_PHY_UNUSED; hisi_hba->devices[i].device_id = i; + hisi_hba->devices[i].dev_status = HISI_SAS_DEV_INIT; } for (i = 0; i < hisi_hba->queue_count; i++) { diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c index e40cc6b3b67be..89160ab3efb05 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c @@ -868,6 +868,7 @@ hisi_sas_device *alloc_dev_quirk_v2_hw(struct domain_device *device) hisi_hba->devices[i].device_id = i; sas_dev = &hisi_hba->devices[i]; + sas_dev->dev_status = HISI_SAS_DEV_INIT; sas_dev->dev_type = device->dev_type; sas_dev->hisi_hba = hisi_hba; sas_dev->sas_device = device; diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c index 9ec8848ec5413..086695a4099fc 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c @@ -129,6 +129,7 @@ #define PHY_CTRL_RESET_MSK (0x1 << PHY_CTRL_RESET_OFF) #define CMD_HDR_PIR_OFF 8 #define CMD_HDR_PIR_MSK (0x1 << CMD_HDR_PIR_OFF) +#define SERDES_CFG (PORT_BASE + 0x1c) #define SL_CFG (PORT_BASE + 0x84) #define AIP_LIMIT (PORT_BASE + 0x90) #define SL_CONTROL (PORT_BASE + 0x94) @@ -181,6 +182,8 @@ #define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF 22 #define CHL_INT2 (PORT_BASE + 0x1bc) #define CHL_INT2_SL_IDAF_TOUT_CONF_OFF 0 +#define CHL_INT2_RX_DISP_ERR_OFF 28 +#define CHL_INT2_RX_CODE_ERR_OFF 29 #define CHL_INT2_RX_INVLD_DW_OFF 30 #define CHL_INT2_STP_LINK_TIMEOUT_OFF 31 #define CHL_INT0_MSK (PORT_BASE + 0x1c0) @@ -523,6 +526,7 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba) } hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE, prog_phy_link_rate); + hisi_sas_phy_write32(hisi_hba, i, SERDES_CFG, 0xffc00); hisi_sas_phy_write32(hisi_hba, i, SAS_RX_TRAIN_TIMER, 0x13e80); hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff); hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff); @@ -544,6 +548,8 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba) hisi_sas_phy_write32(hisi_hba, i, STP_LINK_TIMER, 0x7f7a120); hisi_sas_phy_write32(hisi_hba, i, CON_CFG_DRIVER, 0x2a0a01); hisi_sas_phy_write32(hisi_hba, i, SAS_SSP_CON_TIMER_CFG, 0x32); + hisi_sas_phy_write32(hisi_hba, i, SAS_EC_INT_COAL_TIME, + 0x30f4240); /* used for 12G negotiate */ hisi_sas_phy_write32(hisi_hba, i, COARSETUNE_TIME, 0x1e); hisi_sas_phy_write32(hisi_hba, i, AIP_LIMIT, 0x2ffff); @@ -1344,7 +1350,8 @@ static void prep_abort_v3_hw(struct hisi_hba *hisi_hba, static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba) { - int i, res; + int i; + irqreturn_t res; u32 context, port_id, link_rate; struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; struct asd_sas_phy *sas_phy = &phy->sas_phy; @@ -1575,6 +1582,39 @@ static void handle_chl_int1_v3_hw(struct hisi_hba *hisi_hba, int phy_no) hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT1, irq_value); } +static void phy_get_events_v3_hw(struct hisi_hba *hisi_hba, int phy_no) +{ + struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; + struct asd_sas_phy *sas_phy = &phy->sas_phy; + struct sas_phy *sphy = sas_phy->phy; + unsigned long flags; + u32 reg_value; + + spin_lock_irqsave(&phy->lock, flags); + + /* loss dword sync */ + reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DWS_LOST); + sphy->loss_of_dword_sync_count += reg_value; + + /* phy reset problem */ + reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_RESET_PROB); + sphy->phy_reset_problem_count += reg_value; + + /* invalid dword */ + reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW); + sphy->invalid_dword_count += reg_value; + + /* disparity err */ + reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR); + sphy->running_disparity_error_count += reg_value; + + /* code violation error */ + reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_CODE_ERR); + phy->code_violation_err_count += reg_value; + + spin_unlock_irqrestore(&phy->lock, flags); +} + static void handle_chl_int2_v3_hw(struct hisi_hba *hisi_hba, int phy_no) { u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2_MSK); @@ -1582,6 +1622,9 @@ static void handle_chl_int2_v3_hw(struct hisi_hba *hisi_hba, int phy_no) struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; struct pci_dev *pci_dev = hisi_hba->pci_dev; struct device *dev = hisi_hba->dev; + static const u32 msk = BIT(CHL_INT2_RX_DISP_ERR_OFF) | + BIT(CHL_INT2_RX_CODE_ERR_OFF) | + BIT(CHL_INT2_RX_INVLD_DW_OFF); irq_value &= ~irq_msk; if (!irq_value) @@ -1602,6 +1645,25 @@ static void handle_chl_int2_v3_hw(struct hisi_hba *hisi_hba, int phy_no) hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET); } + if (pci_dev->revision > 0x20 && (irq_value & msk)) { + struct asd_sas_phy *sas_phy = &phy->sas_phy; + struct sas_phy *sphy = sas_phy->phy; + + phy_get_events_v3_hw(hisi_hba, phy_no); + + if (irq_value & BIT(CHL_INT2_RX_INVLD_DW_OFF)) + dev_info(dev, "phy%d invalid dword cnt: %u\n", phy_no, + sphy->invalid_dword_count); + + if (irq_value & BIT(CHL_INT2_RX_CODE_ERR_OFF)) + dev_info(dev, "phy%d code violation cnt: %u\n", phy_no, + phy->code_violation_err_count); + + if (irq_value & BIT(CHL_INT2_RX_DISP_ERR_OFF)) + dev_info(dev, "phy%d disparity error cnt: %u\n", phy_no, + sphy->running_disparity_error_count); + } + if ((irq_value & BIT(CHL_INT2_RX_INVLD_DW_OFF)) && (pci_dev->revision == 0x20)) { u32 reg_value; @@ -2230,31 +2292,6 @@ static u32 get_phys_state_v3_hw(struct hisi_hba *hisi_hba) return hisi_sas_read32(hisi_hba, PHY_STATE); } -static void phy_get_events_v3_hw(struct hisi_hba *hisi_hba, int phy_no) -{ - struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; - struct asd_sas_phy *sas_phy = &phy->sas_phy; - struct sas_phy *sphy = sas_phy->phy; - u32 reg_value; - - /* loss dword sync */ - reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DWS_LOST); - sphy->loss_of_dword_sync_count += reg_value; - - /* phy reset problem */ - reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_RESET_PROB); - sphy->phy_reset_problem_count += reg_value; - - /* invalid dword */ - reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW); - sphy->invalid_dword_count += reg_value; - - /* disparity err */ - reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR); - sphy->running_disparity_error_count += reg_value; - -} - static int disable_host_v3_hw(struct hisi_hba *hisi_hba) { struct device *dev = hisi_hba->dev; diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c index 21309d5b456dd..e893949a3d118 100644 --- a/drivers/scsi/libiscsi.c +++ b/drivers/scsi/libiscsi.c @@ -798,7 +798,7 @@ EXPORT_SYMBOL_GPL(iscsi_conn_send_pdu); * @datalen: len of buffer * * iscsi_cmd_rsp sets up the scsi_cmnd fields based on the PDU and - * then completes the command and task. + * then completes the command and task. called under back_lock **/ static void iscsi_scsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr, struct iscsi_task *task, char *data, @@ -894,6 +894,9 @@ out: * @conn: iscsi connection * @hdr: iscsi pdu * @task: scsi command task + * + * iscsi_data_in_rsp sets up the scsi_cmnd fields based on the data received + * then completes the command and task. called under back_lock **/ static void iscsi_data_in_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr, @@ -978,6 +981,16 @@ static int iscsi_send_nopout(struct iscsi_conn *conn, struct iscsi_nopin *rhdr) return 0; } +/** + * iscsi_nop_out_rsp - SCSI NOP Response processing + * @task: scsi command task + * @nop: the nop structure + * @data: where to put the data + * @datalen: length of data + * + * iscsi_nop_out_rsp handles nop response from use or + * from user space. called under back_lock + **/ static int iscsi_nop_out_rsp(struct iscsi_task *task, struct iscsi_nopin *nop, char *data, int datalen) { @@ -1750,7 +1763,9 @@ int iscsi_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc) return 0; prepd_reject: + spin_lock_bh(&session->back_lock); iscsi_complete_task(task, ISCSI_TASK_REQUEUE_SCSIQ); + spin_unlock_bh(&session->back_lock); reject: spin_unlock_bh(&session->frwd_lock); ISCSI_DBG_SESSION(session, "cmd 0x%x rejected (%d)\n", @@ -1758,7 +1773,9 @@ reject: return SCSI_MLQUEUE_TARGET_BUSY; prepd_fault: + spin_lock_bh(&session->back_lock); iscsi_complete_task(task, ISCSI_TASK_REQUEUE_SCSIQ); + spin_unlock_bh(&session->back_lock); fault: spin_unlock_bh(&session->frwd_lock); ISCSI_DBG_SESSION(session, "iscsi: cmd 0x%x is not queued (%d)\n", @@ -3075,8 +3092,9 @@ fail_mgmt_tasks(struct iscsi_session *session, struct iscsi_conn *conn) state = ISCSI_TASK_ABRT_SESS_RECOV; if (task->state == ISCSI_TASK_PENDING) state = ISCSI_TASK_COMPLETED; + spin_lock_bh(&session->back_lock); iscsi_complete_task(task, state); - + spin_unlock_bh(&session->back_lock); } } diff --git a/drivers/scsi/libiscsi_tcp.c b/drivers/scsi/libiscsi_tcp.c index 9923e9e3b8843..c3fe3f3a78f59 100644 --- a/drivers/scsi/libiscsi_tcp.c +++ b/drivers/scsi/libiscsi_tcp.c @@ -129,12 +129,17 @@ static void iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv) BUG_ON(sg->length == 0); /* + * We always map for the recv path. + * * If the page count is greater than one it is ok to send * to the network layer's zero copy send path. If not we - * have to go the slow sendmsg path. We always map for the - * recv path. + * have to go the slow sendmsg path. + * + * Same goes for slab pages: skb_can_coalesce() allows + * coalescing neighboring slab objects into a single frag which + * triggers one of hardened usercopy checks. */ - if (page_count(sg_page(sg)) >= 1 && !recv) + if (!recv && page_count(sg_page(sg)) >= 1 && !PageSlab(sg_page(sg))) return; if (recv) { diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 3b5873f6751e8..7fcdaed3fa945 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -4090,7 +4090,7 @@ lpfc_new_io_buf(struct lpfc_hba *phba, int num_to_alloc) /* Sanity check to ensure our sizing is right for both SCSI and NVME */ if (sizeof(struct lpfc_io_buf) > LPFC_COMMON_IO_BUF_SZ) { lpfc_printf_log(phba, KERN_ERR, LOG_FCP, - "6426 Common buffer size %ld exceeds %d\n", + "6426 Common buffer size %zd exceeds %d\n", sizeof(struct lpfc_io_buf), LPFC_COMMON_IO_BUF_SZ); return 0; @@ -10052,7 +10052,7 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) { struct pci_dev *pdev = phba->pcidev; unsigned long bar0map_len, bar1map_len, bar2map_len; - int error = -ENODEV; + int error; uint32_t if_type; if (!pdev) @@ -10071,7 +10071,7 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) */ if (pci_read_config_dword(pdev, LPFC_SLI_INTF, &phba->sli4_hba.sli_intf.word0)) { - return error; + return -ENODEV; } /* There is no SLI3 failback for SLI4 devices. */ @@ -10081,7 +10081,7 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) "2894 SLI_INTF reg contents invalid " "sli_intf reg 0x%x\n", phba->sli4_hba.sli_intf.word0); - return error; + return -ENODEV; } if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf); @@ -10105,7 +10105,7 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) dev_printk(KERN_ERR, &pdev->dev, "ioremap failed for SLI4 PCI config " "registers.\n"); - goto out; + return -ENODEV; } phba->pci_bar0_memmap_p = phba->sli4_hba.conf_regs_memmap_p; /* Set up BAR0 PCI config space register memory map */ @@ -10116,7 +10116,7 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) { dev_printk(KERN_ERR, &pdev->dev, "FATAL - No BAR0 mapping for SLI4, if_type 2\n"); - goto out; + return -ENODEV; } phba->sli4_hba.conf_regs_memmap_p = ioremap(phba->pci_bar0_map, bar0map_len); @@ -10124,7 +10124,7 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) dev_printk(KERN_ERR, &pdev->dev, "ioremap failed for SLI4 PCI config " "registers.\n"); - goto out; + return -ENODEV; } lpfc_sli4_bar0_register_memmap(phba, if_type); } @@ -10170,6 +10170,7 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) if (!phba->sli4_hba.drbl_regs_memmap_p) { dev_err(&pdev->dev, "ioremap failed for SLI4 HBA doorbell registers.\n"); + error = -ENOMEM; goto out_iounmap_conf; } phba->pci_bar2_memmap_p = phba->sli4_hba.drbl_regs_memmap_p; @@ -10219,6 +10220,7 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) if (!phba->sli4_hba.dpp_regs_memmap_p) { dev_err(&pdev->dev, "ioremap failed for SLI4 HBA dpp registers.\n"); + error = -ENOMEM; goto out_iounmap_ctrl; } phba->pci_bar4_memmap_p = phba->sli4_hba.dpp_regs_memmap_p; @@ -10249,7 +10251,7 @@ out_iounmap_ctrl: iounmap(phba->sli4_hba.ctrl_regs_memmap_p); out_iounmap_conf: iounmap(phba->sli4_hba.conf_regs_memmap_p); -out: + return error; } @@ -11137,7 +11139,8 @@ lpfc_sli4_hba_unset(struct lpfc_hba *phba) lpfc_sli4_ras_dma_free(phba); /* Stop the SLI4 device port */ - phba->pport->work_port_events = 0; + if (phba->pport) + phba->pport->work_port_events = 0; } /** diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index 55ab9d3ee4ba0..1aa00d2c3f74e 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -965,7 +965,7 @@ lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, struct lpfc_nodelist *ndlp; struct lpfc_nvme_fcpreq_priv *freqpriv; struct lpfc_nvme_lport *lport; - uint32_t code, status, idx, cpu; + uint32_t code, status, idx; uint16_t cid, sqhd, data; uint32_t *ptr; @@ -1138,6 +1138,7 @@ out_err: lpfc_nvme_ktime(phba, lpfc_ncmd); } if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { + uint32_t cpu; idx = lpfc_ncmd->cur_iocbq.hba_wqidx; cpu = smp_processor_id(); if (cpu < LPFC_CHECK_CPU_CNT) { diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index d0817facdae36..57b4a463b5892 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -9881,7 +9881,7 @@ __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number, * The WQE can be either 64 or 128 bytes, */ - lockdep_assert_held(&phba->hbalock); + lockdep_assert_held(&pring->ring_lock); if (piocb->sli4_xritag == NO_XRI) { if (piocb->iocb.ulpCommand == CMD_ABORT_XRI_CN || diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index dace907744a5d..293f5cf524d7a 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -3924,12 +3924,12 @@ megasas_transition_to_ready(struct megasas_instance *instance, int ocr) /* * The cur_state should not last for more than max_wait secs */ - for (i = 0; i < max_wait; i++) { + for (i = 0; i < max_wait * 50; i++) { curr_abs_state = instance->instancet-> read_fw_status_reg(instance); if (abs_state == curr_abs_state) { - msleep(1000); + msleep(20); } else break; } diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c index 2eb1ae721a7d3..f928c4d3a1efe 100644 --- a/drivers/scsi/qla2xxx/qla_attr.c +++ b/drivers/scsi/qla2xxx/qla_attr.c @@ -1652,6 +1652,8 @@ qla2x00_port_speed_store(struct device *dev, struct device_attribute *attr, } rval = kstrtol(buf, 10, &type); + if (rval) + return rval; speed = type; if (type == 40 || type == 80 || type == 160 || type == 320) { diff --git a/drivers/scsi/qla2xxx/qla_dfs.c b/drivers/scsi/qla2xxx/qla_dfs.c index ead17288e2a77..5819a45ac5ef7 100644 --- a/drivers/scsi/qla2xxx/qla_dfs.c +++ b/drivers/scsi/qla2xxx/qla_dfs.c @@ -193,6 +193,8 @@ qla_dfs_tgt_counters_show(struct seq_file *s, void *unused) for (i = 0; i < vha->hw->max_qpairs; i++) { qpair = vha->hw->queue_pair_map[i]; + if (!qpair) + continue; qla_core_sbt_cmd += qpair->tgt_counters.qla_core_sbt_cmd; core_qla_que_buf += qpair->tgt_counters.core_qla_que_buf; qla_core_ret_ctio += qpair->tgt_counters.qla_core_ret_ctio; diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c index 63f8e3c198416..456a41d2e2c6c 100644 --- a/drivers/scsi/qla2xxx/qla_iocb.c +++ b/drivers/scsi/qla2xxx/qla_iocb.c @@ -1132,7 +1132,7 @@ qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp, /* if initiator doing write or target doing read */ if (direction_to_device) { for_each_sg(sgl, sg, tot_dsds, i) { - dma_addr_t sle_phys = sg_phys(sg); + u64 sle_phys = sg_phys(sg); /* If SGE addr + len flips bits in upper 32-bits */ if (MSD(sle_phys + sg->length) ^ MSD(sle_phys)) { @@ -1178,7 +1178,7 @@ qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp, ql_dbg(ql_dbg_tgt + ql_dbg_verbose, vha, 0xe023, "%s: sg[%x] (phys=%llx sglen=%x) ldma_sg_len: %x dif_bundl_len: %x ldma_needed: %x\n", - __func__, i, sg_phys(sg), sglen, ldma_sg_len, + __func__, i, (u64)sg_phys(sg), sglen, ldma_sg_len, difctx->dif_bundl_len, ldma_needed); while (sglen) { diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index 5d9ccbab75815..75ec43aa8df38 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -2764,6 +2764,12 @@ static void pqi_process_raid_io_error(struct pqi_io_request *io_request) sshdr.sense_key == HARDWARE_ERROR && sshdr.asc == 0x3e && sshdr.ascq == 0x1) { + struct pqi_ctrl_info *ctrl_info = shost_to_hba(scmd->device->host); + struct pqi_scsi_dev *device = scmd->device->hostdata; + + if (printk_ratelimit()) + scmd_printk(KERN_ERR, scmd, "received 'logical unit failure' from controller for scsi %d:%d:%d:%d\n", + ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun); pqi_take_device_offline(scmd->device, "RAID"); host_byte = DID_NO_CONNECT; } diff --git a/drivers/scsi/ufs/ufs-hisi.c b/drivers/scsi/ufs/ufs-hisi.c index f2d3df357a973..0e855b5afe82a 100644 --- a/drivers/scsi/ufs/ufs-hisi.c +++ b/drivers/scsi/ufs/ufs-hisi.c @@ -640,7 +640,7 @@ static int ufs_hi3670_init(struct ufs_hba *hba) return 0; } -static struct ufs_hba_variant_ops ufs_hba_hi3660_vops = { +static const struct ufs_hba_variant_ops ufs_hba_hi3660_vops = { .name = "hi3660", .init = ufs_hi3660_init, .link_startup_notify = ufs_hisi_link_startup_notify, @@ -649,7 +649,7 @@ static struct ufs_hba_variant_ops ufs_hba_hi3660_vops = { .resume = ufs_hisi_resume, }; -static struct ufs_hba_variant_ops ufs_hba_hi3670_vops = { +static const struct ufs_hba_variant_ops ufs_hba_hi3670_vops = { .name = "hi3670", .init = ufs_hi3670_init, .link_startup_notify = ufs_hisi_link_startup_notify, @@ -669,13 +669,10 @@ MODULE_DEVICE_TABLE(of, ufs_hisi_of_match); static int ufs_hisi_probe(struct platform_device *pdev) { const struct of_device_id *of_id; - struct ufs_hba_variant_ops *vops; - struct device *dev = &pdev->dev; - of_id = of_match_node(ufs_hisi_of_match, dev->of_node); - vops = (struct ufs_hba_variant_ops *)of_id->data; + of_id = of_match_node(ufs_hisi_of_match, pdev->dev.of_node); - return ufshcd_pltfrm_init(pdev, vops); + return ufshcd_pltfrm_init(pdev, of_id->data); } static int ufs_hisi_remove(struct platform_device *pdev) diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.c b/drivers/scsi/ufs/ufshcd-pltfrm.c index 895a9b5ac9899..27213676329c0 100644 --- a/drivers/scsi/ufs/ufshcd-pltfrm.c +++ b/drivers/scsi/ufs/ufshcd-pltfrm.c @@ -297,7 +297,7 @@ static void ufshcd_init_lanes_per_dir(struct ufs_hba *hba) * Returns 0 on success, non-zero value on failure */ int ufshcd_pltfrm_init(struct platform_device *pdev, - struct ufs_hba_variant_ops *vops) + const struct ufs_hba_variant_ops *vops) { struct ufs_hba *hba; void __iomem *mmio_base; diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.h b/drivers/scsi/ufs/ufshcd-pltfrm.h index df64c41803404..1f29e1fd6d520 100644 --- a/drivers/scsi/ufs/ufshcd-pltfrm.h +++ b/drivers/scsi/ufs/ufshcd-pltfrm.h @@ -17,7 +17,7 @@ #include "ufshcd.h" int ufshcd_pltfrm_init(struct platform_device *pdev, - struct ufs_hba_variant_ops *vops); + const struct ufs_hba_variant_ops *vops); void ufshcd_pltfrm_shutdown(struct platform_device *pdev); #ifdef CONFIG_PM diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index 69ba7445d2b37..ecfa898b9ccc0 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -546,7 +546,7 @@ struct ufs_hba { int nutrs; int nutmrs; u32 ufs_version; - struct ufs_hba_variant_ops *vops; + const struct ufs_hba_variant_ops *vops; void *priv; unsigned int irq; bool is_irq_enabled; diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c index 1a6f150cd2d87..8af01777d09c7 100644 --- a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -586,7 +586,6 @@ static int virtscsi_device_reset(struct scsi_cmnd *sc) return FAILED; memset(cmd, 0, sizeof(*cmd)); - cmd->sc = sc; cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){ .type = VIRTIO_SCSI_T_TMF, .subtype = cpu_to_virtio32(vscsi->vdev, @@ -645,7 +644,6 @@ static int virtscsi_abort(struct scsi_cmnd *sc) return FAILED; memset(cmd, 0, sizeof(*cmd)); - cmd->sc = sc; cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){ .type = VIRTIO_SCSI_T_TMF, .subtype = VIRTIO_SCSI_T_TMF_ABORT_TASK, diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c index 5831e0eecea12..9704b135a7bc5 100644 --- a/drivers/target/target_core_user.c +++ b/drivers/target/target_core_user.c @@ -1663,7 +1663,7 @@ static void tcmu_dev_kref_release(struct kref *kref) WARN_ON(!all_expired); tcmu_blocks_release(&udev->data_blocks, 0, udev->dbi_max + 1); - kfree(udev->data_bitmap); + bitmap_free(udev->data_bitmap); mutex_unlock(&udev->cmdr_lock); call_rcu(&dev->rcu_head, tcmu_dev_call_rcu); @@ -1794,11 +1794,12 @@ static int tcmu_netlink_event_send(struct tcmu_dev *udev, ret = genlmsg_multicast_allns(&tcmu_genl_family, skb, 0, TCMU_MCGRP_CONFIG, GFP_KERNEL); - /* We don't care if no one is listening */ - if (ret == -ESRCH) - ret = 0; - if (!ret) - ret = tcmu_wait_genl_cmd_reply(udev); + + /* Wait during an add as the listener may not be up yet */ + if (ret == 0 || + (ret == -ESRCH && cmd == TCMU_CMD_ADDED_DEVICE)) + return tcmu_wait_genl_cmd_reply(udev); + return ret; } @@ -1870,9 +1871,7 @@ static int tcmu_configure_device(struct se_device *dev) info = &udev->uio_info; mutex_lock(&udev->cmdr_lock); - udev->data_bitmap = kcalloc(BITS_TO_LONGS(udev->max_blocks), - sizeof(unsigned long), - GFP_KERNEL); + udev->data_bitmap = bitmap_zalloc(udev->max_blocks, GFP_KERNEL); mutex_unlock(&udev->cmdr_lock); if (!udev->data_bitmap) { ret = -ENOMEM; @@ -1959,7 +1958,7 @@ err_register: vfree(udev->mb_addr); udev->mb_addr = NULL; err_vzalloc: - kfree(udev->data_bitmap); + bitmap_free(udev->data_bitmap); udev->data_bitmap = NULL; err_bitmap_alloc: kfree(info->name); diff --git a/drivers/tty/serial/8250/8250_lpss.c b/drivers/tty/serial/8250/8250_lpss.c index 98dbc796353fb..53ca9ba6ab4b1 100644 --- a/drivers/tty/serial/8250/8250_lpss.c +++ b/drivers/tty/serial/8250/8250_lpss.c @@ -153,7 +153,6 @@ static int byt_serial_setup(struct lpss8250 *lpss, struct uart_port *port) #ifdef CONFIG_SERIAL_8250_DMA static const struct dw_dma_platform_data qrk_serial_dma_pdata = { .nr_channels = 2, - .is_private = true, .chan_allocation_order = CHAN_ALLOCATION_ASCENDING, .chan_priority = CHAN_PRIORITY_ASCENDING, .block_size = 4095, diff --git a/drivers/video/fbdev/aty/radeon_pm.c b/drivers/video/fbdev/aty/radeon_pm.c index e695adb0e5733..2dc5703eac519 100644 --- a/drivers/video/fbdev/aty/radeon_pm.c +++ b/drivers/video/fbdev/aty/radeon_pm.c @@ -2844,8 +2844,8 @@ void radeonfb_pm_init(struct radeonfb_info *rinfo, int dynclk, int ignore_devlis * in some desktop G4s), Via (M9+ chip on iBook G4) and * Snowy (M11 chip on iBook G4 manufactured after July 2005) */ - if (!strcmp(rinfo->of_node->name, "ATY,JasperParent") || - !strcmp(rinfo->of_node->name, "ATY,SnowyParent")) { + if (of_node_name_eq(rinfo->of_node, "ATY,JasperParent") || + of_node_name_eq(rinfo->of_node, "ATY,SnowyParent")) { rinfo->reinit_func = radeon_reinitialize_M10; rinfo->pm_mode |= radeon_pm_off; } @@ -2855,7 +2855,7 @@ void radeonfb_pm_init(struct radeonfb_info *rinfo, int dynclk, int ignore_devlis rinfo->pm_mode |= radeon_pm_off; } #endif - if (!strcmp(rinfo->of_node->name, "ATY,ViaParent")) { + if (of_node_name_eq(rinfo->of_node, "ATY,ViaParent")) { rinfo->reinit_func = radeon_reinitialize_M9P; rinfo->pm_mode |= radeon_pm_off; } diff --git a/drivers/video/fbdev/cg14.c b/drivers/video/fbdev/cg14.c index 9af54c2368fdb..a6dce1a78490a 100644 --- a/drivers/video/fbdev/cg14.c +++ b/drivers/video/fbdev/cg14.c @@ -486,8 +486,8 @@ static int cg14_probe(struct platform_device *op) info->var.xres); info->fix.smem_len = PAGE_ALIGN(linebytes * info->var.yres); - if (!strcmp(dp->parent->name, "sbus") || - !strcmp(dp->parent->name, "sbi")) { + if (of_node_name_eq(dp->parent, "sbus") || + of_node_name_eq(dp->parent, "sbi")) { info->fix.smem_start = op->resource[0].start; par->iospace = op->resource[0].flags & IORESOURCE_BITS; } else { diff --git a/drivers/video/fbdev/cg3.c b/drivers/video/fbdev/cg3.c index 1bd95b02f3aa4..6d42def8436b2 100644 --- a/drivers/video/fbdev/cg3.c +++ b/drivers/video/fbdev/cg3.c @@ -369,7 +369,7 @@ static int cg3_probe(struct platform_device *op) info->var.red.length = 8; info->var.green.length = 8; info->var.blue.length = 8; - if (!strcmp(dp->name, "cgRDI")) + if (of_node_name_eq(dp, "cgRDI")) par->flags |= CG3_FLAG_RDI; if (par->flags & CG3_FLAG_RDI) cg3_rdi_maybe_fixup_var(&info->var, dp); diff --git a/drivers/video/fbdev/chipsfb.c b/drivers/video/fbdev/chipsfb.c index 40182ed856483..ca549e1532e65 100644 --- a/drivers/video/fbdev/chipsfb.c +++ b/drivers/video/fbdev/chipsfb.c @@ -349,7 +349,7 @@ static void init_chips(struct fb_info *p, unsigned long addr) static int chipsfb_pci_init(struct pci_dev *dp, const struct pci_device_id *ent) { struct fb_info *p; - unsigned long addr, size; + unsigned long addr; unsigned short cmd; int rc = -ENODEV; @@ -361,7 +361,6 @@ static int chipsfb_pci_init(struct pci_dev *dp, const struct pci_device_id *ent) if ((dp->resource[0].flags & IORESOURCE_MEM) == 0) goto err_disable; addr = pci_resource_start(dp, 0); - size = pci_resource_len(dp, 0); if (addr == 0) goto err_disable; diff --git a/drivers/video/fbdev/core/fb_cmdline.c b/drivers/video/fbdev/core/fb_cmdline.c index 39509ccd92f1e..3b5bd666b9525 100644 --- a/drivers/video/fbdev/core/fb_cmdline.c +++ b/drivers/video/fbdev/core/fb_cmdline.c @@ -75,36 +75,33 @@ EXPORT_SYMBOL(fb_get_options); * NOTE: This function is a __setup and __init function. * It only stores the options. Drivers have to call * fb_get_options() as necessary. - * - * Returns zero. - * */ static int __init video_setup(char *options) { - int i, global = 0; - if (!options || !*options) - global = 1; + goto out; - if (!global && !strncmp(options, "ofonly", 6)) { + if (!strncmp(options, "ofonly", 6)) { ofonly = 1; - global = 1; + goto out; } - if (!global && !strchr(options, ':')) { - fb_mode_option = options; - global = 1; - } + if (strchr(options, ':')) { + /* named */ + int i; - if (!global) { for (i = 0; i < FB_MAX; i++) { if (video_options[i] == NULL) { video_options[i] = options; break; } } + } else { + /* global */ + fb_mode_option = options; } +out: return 1; } __setup("video=", video_setup); diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index bfa1360ec7504..cd059a801662e 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -656,11 +656,14 @@ static void fbcon_prepare_logo(struct vc_data *vc, struct fb_info *info, kfree(save); } + if (logo_shown == FBCON_LOGO_DONTSHOW) + return; + if (logo_lines > vc->vc_bottom) { logo_shown = FBCON_LOGO_CANSHOW; printk(KERN_INFO "fbcon_init: disable boot-logo (boot-logo bigger than screen).\n"); - } else if (logo_shown != FBCON_LOGO_DONTSHOW) { + } else { logo_shown = FBCON_LOGO_DRAW; vc->vc_top = logo_lines; } @@ -999,7 +1002,7 @@ static const char *fbcon_startup(void) if (!softback_buf) { softback_buf = (unsigned long) - kmalloc(fbcon_softback_size, + kvmalloc(fbcon_softback_size, GFP_KERNEL); if (!softback_buf) { fbcon_softback_size = 0; @@ -1008,7 +1011,7 @@ static const char *fbcon_startup(void) } } else { if (softback_buf) { - kfree((void *) softback_buf); + kvfree((void *) softback_buf); softback_buf = 0; softback_top = 0; } @@ -1066,6 +1069,9 @@ static void fbcon_init(struct vc_data *vc, int init) cap = info->flags; + if (console_loglevel <= CONSOLE_LOGLEVEL_QUIET) + logo_shown = FBCON_LOGO_DONTSHOW; + if (vc != svc || logo_shown == FBCON_LOGO_DONTSHOW || (info->fix.type == FB_TYPE_TEXT)) logo = 0; @@ -3672,7 +3678,7 @@ static void fbcon_exit(void) } #endif - kfree((void *)softback_buf); + kvfree((void *)softback_buf); softback_buf = 0UL; for_each_registered_fb(i) { diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c index cb43a2258c518..4721491e6c8cf 100644 --- a/drivers/video/fbdev/core/fbmem.c +++ b/drivers/video/fbdev/core/fbmem.c @@ -431,6 +431,9 @@ static void fb_do_show_logo(struct fb_info *info, struct fb_image *image, { unsigned int x; + if (image->width > info->var.xres || image->height > info->var.yres) + return; + if (rotate == FB_ROTATE_UR) { for (x = 0; x < num && image->dx + image->width <= info->var.xres; diff --git a/drivers/video/fbdev/core/fbmon.c b/drivers/video/fbdev/core/fbmon.c index dd3128990776e..3558a70a66640 100644 --- a/drivers/video/fbdev/core/fbmon.c +++ b/drivers/video/fbdev/core/fbmon.c @@ -978,6 +978,8 @@ void fb_edid_to_monspecs(unsigned char *edid, struct fb_monspecs *specs) get_monspecs(edid, specs); specs->modedb = fb_create_modedb(edid, &specs->modedb_len, specs); + if (!specs->modedb) + return; /* * Workaround for buggy EDIDs that sets that the first diff --git a/drivers/video/fbdev/ffb.c b/drivers/video/fbdev/ffb.c index 6b1915872af16..b7aee0c427a87 100644 --- a/drivers/video/fbdev/ffb.c +++ b/drivers/video/fbdev/ffb.c @@ -944,7 +944,7 @@ static int ffb_probe(struct platform_device *op) info->var.accel_flags = FB_ACCELF_TEXT; - if (!strcmp(dp->name, "SUNW,afb")) + if (of_node_name_eq(dp, "SUNW,afb")) par->flags |= FFB_FLAG_AFB; par->board_type = of_getintprop_default(dp, "board_type", 0); diff --git a/drivers/video/fbdev/geode/gxfb_core.c b/drivers/video/fbdev/geode/gxfb_core.c index f4f76373b2a87..b1906cf5a8f05 100644 --- a/drivers/video/fbdev/geode/gxfb_core.c +++ b/drivers/video/fbdev/geode/gxfb_core.c @@ -33,6 +33,8 @@ #include <linux/pci.h> #include <linux/cs5535.h> +#include <asm/olpc.h> + #include "gxfb.h" static char *mode_option; @@ -107,9 +109,6 @@ static struct fb_videomode gx_modedb[] = { FB_VMODE_NONINTERLACED, FB_MODE_IS_VESA }, }; -#ifdef CONFIG_OLPC -#include <asm/olpc.h> - static struct fb_videomode gx_dcon_modedb[] = { /* The only mode the DCON has is 1200x900 */ { NULL, 50, 1200, 900, 17460, 24, 8, 4, 5, 8, 3, @@ -128,14 +127,6 @@ static void get_modedb(struct fb_videomode **modedb, unsigned int *size) } } -#else -static void get_modedb(struct fb_videomode **modedb, unsigned int *size) -{ - *modedb = (struct fb_videomode *) gx_modedb; - *size = ARRAY_SIZE(gx_modedb); -} -#endif - static int gxfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info) { if (var->xres > 1600 || var->yres > 1200) diff --git a/drivers/video/fbdev/geode/lxfb_core.c b/drivers/video/fbdev/geode/lxfb_core.c index 138da6cb6cbcd..17ab905811b1e 100644 --- a/drivers/video/fbdev/geode/lxfb_core.c +++ b/drivers/video/fbdev/geode/lxfb_core.c @@ -23,6 +23,8 @@ #include <linux/pci.h> #include <linux/uaccess.h> +#include <asm/olpc.h> + #include "lxfb.h" static char *mode_option; @@ -216,9 +218,6 @@ static struct fb_videomode geode_modedb[] = { 0, FB_VMODE_NONINTERLACED, 0 }, }; -#ifdef CONFIG_OLPC -#include <asm/olpc.h> - static struct fb_videomode olpc_dcon_modedb[] = { /* The only mode the DCON has is 1200x900 */ { NULL, 50, 1200, 900, 17460, 24, 8, 4, 5, 8, 3, @@ -237,14 +236,6 @@ static void get_modedb(struct fb_videomode **modedb, unsigned int *size) } } -#else -static void get_modedb(struct fb_videomode **modedb, unsigned int *size) -{ - *modedb = (struct fb_videomode *) geode_modedb; - *size = ARRAY_SIZE(geode_modedb); -} -#endif - static int lxfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info) { if (var->xres > 1920 || var->yres > 1440) diff --git a/drivers/video/fbdev/imsttfb.c b/drivers/video/fbdev/imsttfb.c index 5d9670daf60ec..4b9615e4ce746 100644 --- a/drivers/video/fbdev/imsttfb.c +++ b/drivers/video/fbdev/imsttfb.c @@ -1497,8 +1497,8 @@ static int imsttfb_probe(struct pci_dev *pdev, const struct pci_device_id *ent) switch (pdev->device) { case PCI_DEVICE_ID_IMS_TT128: /* IMS,tt128mbA */ par->ramdac = IBM; - if (dp && ((strcmp(dp->name, "IMS,tt128mb8") == 0) || - (strcmp(dp->name, "IMS,tt128mb8A") == 0))) + if (of_node_name_eq(dp, "IMS,tt128mb8") || + of_node_name_eq(dp, "IMS,tt128mb8A")) par->ramdac = TVP; break; case PCI_DEVICE_ID_IMS_TT3D: /* IMS,tt3d */ diff --git a/drivers/video/fbdev/mbx/mbxdebugfs.c b/drivers/video/fbdev/mbx/mbxdebugfs.c index 2bd328883178d..09af721638fb6 100644 --- a/drivers/video/fbdev/mbx/mbxdebugfs.c +++ b/drivers/video/fbdev/mbx/mbxdebugfs.c @@ -211,36 +211,22 @@ static const struct file_operations misc_fops = { static void mbxfb_debugfs_init(struct fb_info *fbi) { struct mbxfb_info *mfbi = fbi->par; - struct mbxfb_debugfs_data *dbg; - - dbg = kzalloc(sizeof(struct mbxfb_debugfs_data), GFP_KERNEL); - mfbi->debugfs_data = dbg; - - dbg->dir = debugfs_create_dir("mbxfb", NULL); - dbg->sysconf = debugfs_create_file("sysconf", 0444, dbg->dir, - fbi, &sysconf_fops); - dbg->clock = debugfs_create_file("clock", 0444, dbg->dir, - fbi, &clock_fops); - dbg->display = debugfs_create_file("display", 0444, dbg->dir, - fbi, &display_fops); - dbg->gsctl = debugfs_create_file("gsctl", 0444, dbg->dir, - fbi, &gsctl_fops); - dbg->sdram = debugfs_create_file("sdram", 0444, dbg->dir, - fbi, &sdram_fops); - dbg->misc = debugfs_create_file("misc", 0444, dbg->dir, - fbi, &misc_fops); + struct dentry *dir; + + dir = debugfs_create_dir("mbxfb", NULL); + mfbi->debugfs_dir = dir; + + debugfs_create_file("sysconf", 0444, dir, fbi, &sysconf_fops); + debugfs_create_file("clock", 0444, dir, fbi, &clock_fops); + debugfs_create_file("display", 0444, dir, fbi, &display_fops); + debugfs_create_file("gsctl", 0444, dir, fbi, &gsctl_fops); + debugfs_create_file("sdram", 0444, dir, fbi, &sdram_fops); + debugfs_create_file("misc", 0444, dir, fbi, &misc_fops); } static void mbxfb_debugfs_remove(struct fb_info *fbi) { struct mbxfb_info *mfbi = fbi->par; - struct mbxfb_debugfs_data *dbg = mfbi->debugfs_data; - - debugfs_remove(dbg->misc); - debugfs_remove(dbg->sdram); - debugfs_remove(dbg->gsctl); - debugfs_remove(dbg->display); - debugfs_remove(dbg->clock); - debugfs_remove(dbg->sysconf); - debugfs_remove(dbg->dir); + + debugfs_remove_recursive(mfbi->debugfs_dir); } diff --git a/drivers/video/fbdev/mbx/mbxfb.c b/drivers/video/fbdev/mbx/mbxfb.c index 539b85da08973..6ded480a69b4a 100644 --- a/drivers/video/fbdev/mbx/mbxfb.c +++ b/drivers/video/fbdev/mbx/mbxfb.c @@ -74,7 +74,7 @@ struct mbxfb_info { u32 pseudo_palette[MAX_PALETTES]; #ifdef CONFIG_FB_MBX_DEBUG - void *debugfs_data; + struct dentry *debugfs_dir; #endif }; diff --git a/drivers/video/fbdev/offb.c b/drivers/video/fbdev/offb.c index 057d3cdef92e6..fbc6eafb63c77 100644 --- a/drivers/video/fbdev/offb.c +++ b/drivers/video/fbdev/offb.c @@ -141,6 +141,7 @@ static int offb_setcolreg(u_int regno, u_int red, u_int green, u_int blue, /* Clear PALETTE_ACCESS_CNTL in DAC_CNTL */ out_le32(par->cmap_adr + 0x58, in_le32(par->cmap_adr + 0x58) & ~0x20); + /* fall through */ case cmap_r128: /* Set palette index & data */ out_8(par->cmap_adr + 0xb0, regno); @@ -210,6 +211,7 @@ static int offb_blank(int blank, struct fb_info *info) /* Clear PALETTE_ACCESS_CNTL in DAC_CNTL */ out_le32(par->cmap_adr + 0x58, in_le32(par->cmap_adr + 0x58) & ~0x20); + /* fall through */ case cmap_r128: /* Set palette index & data */ out_8(par->cmap_adr + 0xb0, i); @@ -646,7 +648,7 @@ static void __init offb_init_nodriver(struct device_node *dp, int no_real_node) } #endif /* kludge for valkyrie */ - if (strcmp(dp->name, "valkyrie") == 0) + if (of_node_name_eq(dp, "valkyrie")) address += 0x1000; offb_init_fb(no_real_node ? "bootx" : NULL, width, height, depth, pitch, address, diff --git a/drivers/video/fbdev/omap2/omapfb/dss/core.c b/drivers/video/fbdev/omap2/omapfb/dss/core.c index b4bcf3a4a6475..b5956a1a30d44 100644 --- a/drivers/video/fbdev/omap2/omapfb/dss/core.c +++ b/drivers/video/fbdev/omap2/omapfb/dss/core.c @@ -110,19 +110,12 @@ DEFINE_SHOW_ATTRIBUTE(dss); static struct dentry *dss_debugfs_dir; -static int dss_initialize_debugfs(void) +static void dss_initialize_debugfs(void) { dss_debugfs_dir = debugfs_create_dir("omapdss", NULL); - if (IS_ERR(dss_debugfs_dir)) { - int err = PTR_ERR(dss_debugfs_dir); - dss_debugfs_dir = NULL; - return err; - } debugfs_create_file("clk", S_IRUGO, dss_debugfs_dir, &dss_debug_dump_clocks, &dss_fops); - - return 0; } static void dss_uninitialize_debugfs(void) @@ -130,26 +123,19 @@ static void dss_uninitialize_debugfs(void) debugfs_remove_recursive(dss_debugfs_dir); } -int dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *)) +void dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *)) { - struct dentry *d; - - d = debugfs_create_file(name, S_IRUGO, dss_debugfs_dir, - write, &dss_fops); - - return PTR_ERR_OR_ZERO(d); + debugfs_create_file(name, S_IRUGO, dss_debugfs_dir, write, &dss_fops); } #else /* CONFIG_FB_OMAP2_DSS_DEBUGFS */ -static inline int dss_initialize_debugfs(void) +static inline void dss_initialize_debugfs(void) { - return 0; } static inline void dss_uninitialize_debugfs(void) { } -int dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *)) +void dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *)) { - return 0; } #endif /* CONFIG_FB_OMAP2_DSS_DEBUGFS */ @@ -182,15 +168,11 @@ static struct notifier_block omap_dss_pm_notif_block = { static int __init omap_dss_probe(struct platform_device *pdev) { - int r; - core.pdev = pdev; dss_features_init(omapdss_get_version()); - r = dss_initialize_debugfs(); - if (r) - goto err_debugfs; + dss_initialize_debugfs(); if (def_disp_name) core.default_display_name = def_disp_name; @@ -198,10 +180,6 @@ static int __init omap_dss_probe(struct platform_device *pdev) register_pm_notifier(&omap_dss_pm_notif_block); return 0; - -err_debugfs: - - return r; } static int omap_dss_remove(struct platform_device *pdev) diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c b/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c index f1eb8b0f8a2aa..5ce893c1923d2 100644 --- a/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c +++ b/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c @@ -60,7 +60,7 @@ omapdss_of_get_next_port(const struct device_node *parent, return NULL; } prev = port; - } while (of_node_cmp(port->name, "port") != 0); + } while (!of_node_name_eq(port, "port")); of_node_put(ports); } @@ -83,7 +83,7 @@ omapdss_of_get_next_endpoint(const struct device_node *parent, if (!ep) return NULL; prev = ep; - } while (of_node_cmp(ep->name, "endpoint") != 0); + } while (!of_node_name_eq(ep, "endpoint")); return ep; } diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss.h b/drivers/video/fbdev/omap2/omapfb/dss/dss.h index a3cc0ca8f9d24..b1a3544941447 100644 --- a/drivers/video/fbdev/omap2/omapfb/dss/dss.h +++ b/drivers/video/fbdev/omap2/omapfb/dss/dss.h @@ -214,7 +214,7 @@ struct platform_device *dss_get_core_pdev(void); int dss_dsi_enable_pads(int dsi_id, unsigned lane_mask); void dss_dsi_disable_pads(int dsi_id, unsigned lane_mask); int dss_set_min_bus_tput(struct device *dev, unsigned long tput); -int dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *)); +void dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *)); /* display */ int dss_suspend_all_devices(void); diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4_core.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4_core.c index fa72e735dad2f..d146793dd044a 100644 --- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4_core.c +++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4_core.c @@ -712,7 +712,7 @@ int hdmi4_audio_config(struct hdmi_core_data *core, struct hdmi_wp_data *wp, else acore.i2s_cfg.justification = HDMI_AUDIO_JUSTIFY_RIGHT; /* - * The I2S input word length is twice the lenght given in the IEC-60958 + * The I2S input word length is twice the length given in the IEC-60958 * status word. If the word size is greater than * 20 bits, increment by one. */ diff --git a/drivers/video/fbdev/ssd1307fb.c b/drivers/video/fbdev/ssd1307fb.c index 4061a20cfe24a..3b361bc9feb8a 100644 --- a/drivers/video/fbdev/ssd1307fb.c +++ b/drivers/video/fbdev/ssd1307fb.c @@ -667,10 +667,10 @@ static int ssd1307fb_probe(struct i2c_client *client, if (par->reset) { /* Reset the screen */ - gpiod_set_value_cansleep(par->reset, 0); - udelay(4); gpiod_set_value_cansleep(par->reset, 1); udelay(4); + gpiod_set_value_cansleep(par->reset, 0); + udelay(4); } if (par->vbat_reg) { diff --git a/drivers/video/fbdev/via/viafbdev.c b/drivers/video/fbdev/via/viafbdev.c index 7bb7e90b8f006..bdf5a0ea876de 100644 --- a/drivers/video/fbdev/via/viafbdev.c +++ b/drivers/video/fbdev/via/viafbdev.c @@ -2110,7 +2110,7 @@ MODULE_PARM_DESC(viafb_lcd_panel_id, module_param(viafb_lcd_dsp_method, int, S_IRUSR); MODULE_PARM_DESC(viafb_lcd_dsp_method, - "Set Flat Panel display scaling method.(Default=Expandsion)"); + "Set Flat Panel display scaling method.(Default=Expansion)"); module_param(viafb_SAMM_ON, int, S_IRUSR); MODULE_PARM_DESC(viafb_SAMM_ON, diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 39b229f9e256e..d37dd5bb7a8fb 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -604,6 +604,7 @@ int alloc_xenballooned_pages(int nr_pages, struct page **pages) while (pgno < nr_pages) { page = balloon_retrieve(true); if (page) { + __ClearPageOffline(page); pages[pgno++] = page; #ifdef CONFIG_XEN_HAVE_PVMMU /* @@ -645,8 +646,10 @@ void free_xenballooned_pages(int nr_pages, struct page **pages) mutex_lock(&balloon_mutex); for (i = 0; i < nr_pages; i++) { - if (pages[i]) + if (pages[i]) { + __SetPageOffline(pages[i]); balloon_append(pages[i]); + } } balloon_stats.target_unpopulated -= nr_pages; diff --git a/fs/9p/v9fs_vfs.h b/fs/9p/v9fs_vfs.h index 5a0db6dec8d1f..aaee1e6584e65 100644 --- a/fs/9p/v9fs_vfs.h +++ b/fs/9p/v9fs_vfs.h @@ -40,6 +40,9 @@ */ #define P9_LOCK_TIMEOUT (30*HZ) +/* flags for v9fs_stat2inode() & v9fs_stat2inode_dotl() */ +#define V9FS_STAT2INODE_KEEP_ISIZE 1 + extern struct file_system_type v9fs_fs_type; extern const struct address_space_operations v9fs_addr_operations; extern const struct file_operations v9fs_file_operations; @@ -61,8 +64,10 @@ int v9fs_init_inode(struct v9fs_session_info *v9ses, struct inode *inode, umode_t mode, dev_t); void v9fs_evict_inode(struct inode *inode); ino_t v9fs_qid2ino(struct p9_qid *qid); -void v9fs_stat2inode(struct p9_wstat *, struct inode *, struct super_block *); -void v9fs_stat2inode_dotl(struct p9_stat_dotl *, struct inode *); +void v9fs_stat2inode(struct p9_wstat *stat, struct inode *inode, + struct super_block *sb, unsigned int flags); +void v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode, + unsigned int flags); int v9fs_dir_release(struct inode *inode, struct file *filp); int v9fs_file_open(struct inode *inode, struct file *file); void v9fs_inode2stat(struct inode *inode, struct p9_wstat *stat); @@ -83,4 +88,18 @@ static inline void v9fs_invalidate_inode_attr(struct inode *inode) } int v9fs_open_to_dotl_flags(int flags); + +static inline void v9fs_i_size_write(struct inode *inode, loff_t i_size) +{ + /* + * 32-bit need the lock, concurrent updates could break the + * sequences and make i_size_read() loop forever. + * 64-bit updates are atomic and can skip the locking. + */ + if (sizeof(i_size) > sizeof(long)) + spin_lock(&inode->i_lock); + i_size_write(inode, i_size); + if (sizeof(i_size) > sizeof(long)) + spin_unlock(&inode->i_lock); +} #endif diff --git a/fs/9p/vfs_file.c b/fs/9p/vfs_file.c index a25efa782fccb..9a1125305d842 100644 --- a/fs/9p/vfs_file.c +++ b/fs/9p/vfs_file.c @@ -446,7 +446,11 @@ v9fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) i_size = i_size_read(inode); if (iocb->ki_pos > i_size) { inode_add_bytes(inode, iocb->ki_pos - i_size); - i_size_write(inode, iocb->ki_pos); + /* + * Need to serialize against i_size_write() in + * v9fs_stat2inode() + */ + v9fs_i_size_write(inode, iocb->ki_pos); } return retval; } diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index 85ff859d3af5f..72b779bc09422 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -538,7 +538,7 @@ static struct inode *v9fs_qid_iget(struct super_block *sb, if (retval) goto error; - v9fs_stat2inode(st, inode, sb); + v9fs_stat2inode(st, inode, sb, 0); v9fs_cache_inode_get_cookie(inode); unlock_new_inode(inode); return inode; @@ -1092,7 +1092,7 @@ v9fs_vfs_getattr(const struct path *path, struct kstat *stat, if (IS_ERR(st)) return PTR_ERR(st); - v9fs_stat2inode(st, d_inode(dentry), dentry->d_sb); + v9fs_stat2inode(st, d_inode(dentry), dentry->d_sb, 0); generic_fillattr(d_inode(dentry), stat); p9stat_free(st); @@ -1170,12 +1170,13 @@ static int v9fs_vfs_setattr(struct dentry *dentry, struct iattr *iattr) * @stat: Plan 9 metadata (mistat) structure * @inode: inode to populate * @sb: superblock of filesystem + * @flags: control flags (e.g. V9FS_STAT2INODE_KEEP_ISIZE) * */ void v9fs_stat2inode(struct p9_wstat *stat, struct inode *inode, - struct super_block *sb) + struct super_block *sb, unsigned int flags) { umode_t mode; char ext[32]; @@ -1216,10 +1217,11 @@ v9fs_stat2inode(struct p9_wstat *stat, struct inode *inode, mode = p9mode2perm(v9ses, stat); mode |= inode->i_mode & ~S_IALLUGO; inode->i_mode = mode; - i_size_write(inode, stat->length); + if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE)) + v9fs_i_size_write(inode, stat->length); /* not real number of blocks, but 512 byte ones ... */ - inode->i_blocks = (i_size_read(inode) + 512 - 1) >> 9; + inode->i_blocks = (stat->length + 512 - 1) >> 9; v9inode->cache_validity &= ~V9FS_INO_INVALID_ATTR; } @@ -1416,9 +1418,9 @@ int v9fs_refresh_inode(struct p9_fid *fid, struct inode *inode) { int umode; dev_t rdev; - loff_t i_size; struct p9_wstat *st; struct v9fs_session_info *v9ses; + unsigned int flags; v9ses = v9fs_inode2v9ses(inode); st = p9_client_stat(fid); @@ -1431,16 +1433,13 @@ int v9fs_refresh_inode(struct p9_fid *fid, struct inode *inode) if ((inode->i_mode & S_IFMT) != (umode & S_IFMT)) goto out; - spin_lock(&inode->i_lock); /* * We don't want to refresh inode->i_size, * because we may have cached data */ - i_size = inode->i_size; - v9fs_stat2inode(st, inode, inode->i_sb); - if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) - inode->i_size = i_size; - spin_unlock(&inode->i_lock); + flags = (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) ? + V9FS_STAT2INODE_KEEP_ISIZE : 0; + v9fs_stat2inode(st, inode, inode->i_sb, flags); out: p9stat_free(st); kfree(st); diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c index 4823e1c469994..a950a927a626f 100644 --- a/fs/9p/vfs_inode_dotl.c +++ b/fs/9p/vfs_inode_dotl.c @@ -143,7 +143,7 @@ static struct inode *v9fs_qid_iget_dotl(struct super_block *sb, if (retval) goto error; - v9fs_stat2inode_dotl(st, inode); + v9fs_stat2inode_dotl(st, inode, 0); v9fs_cache_inode_get_cookie(inode); retval = v9fs_get_acl(inode, fid); if (retval) @@ -496,7 +496,7 @@ v9fs_vfs_getattr_dotl(const struct path *path, struct kstat *stat, if (IS_ERR(st)) return PTR_ERR(st); - v9fs_stat2inode_dotl(st, d_inode(dentry)); + v9fs_stat2inode_dotl(st, d_inode(dentry), 0); generic_fillattr(d_inode(dentry), stat); /* Change block size to what the server returned */ stat->blksize = st->st_blksize; @@ -607,11 +607,13 @@ int v9fs_vfs_setattr_dotl(struct dentry *dentry, struct iattr *iattr) * v9fs_stat2inode_dotl - populate an inode structure with stat info * @stat: stat structure * @inode: inode to populate + * @flags: ctrl flags (e.g. V9FS_STAT2INODE_KEEP_ISIZE) * */ void -v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode) +v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode, + unsigned int flags) { umode_t mode; struct v9fs_inode *v9inode = V9FS_I(inode); @@ -631,7 +633,8 @@ v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode) mode |= inode->i_mode & ~S_IALLUGO; inode->i_mode = mode; - i_size_write(inode, stat->st_size); + if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE)) + v9fs_i_size_write(inode, stat->st_size); inode->i_blocks = stat->st_blocks; } else { if (stat->st_result_mask & P9_STATS_ATIME) { @@ -661,8 +664,9 @@ v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode) } if (stat->st_result_mask & P9_STATS_RDEV) inode->i_rdev = new_decode_dev(stat->st_rdev); - if (stat->st_result_mask & P9_STATS_SIZE) - i_size_write(inode, stat->st_size); + if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE) && + stat->st_result_mask & P9_STATS_SIZE) + v9fs_i_size_write(inode, stat->st_size); if (stat->st_result_mask & P9_STATS_BLOCKS) inode->i_blocks = stat->st_blocks; } @@ -928,9 +932,9 @@ v9fs_vfs_get_link_dotl(struct dentry *dentry, int v9fs_refresh_inode_dotl(struct p9_fid *fid, struct inode *inode) { - loff_t i_size; struct p9_stat_dotl *st; struct v9fs_session_info *v9ses; + unsigned int flags; v9ses = v9fs_inode2v9ses(inode); st = p9_client_getattr_dotl(fid, P9_STATS_ALL); @@ -942,16 +946,13 @@ int v9fs_refresh_inode_dotl(struct p9_fid *fid, struct inode *inode) if ((inode->i_mode & S_IFMT) != (st->st_mode & S_IFMT)) goto out; - spin_lock(&inode->i_lock); /* * We don't want to refresh inode->i_size, * because we may have cached data */ - i_size = inode->i_size; - v9fs_stat2inode_dotl(st, inode); - if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) - inode->i_size = i_size; - spin_unlock(&inode->i_lock); + flags = (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) ? + V9FS_STAT2INODE_KEEP_ISIZE : 0; + v9fs_stat2inode_dotl(st, inode, flags); out: kfree(st); return 0; diff --git a/fs/9p/vfs_super.c b/fs/9p/vfs_super.c index 10d3bd3f534bc..d13d35cf69c0e 100644 --- a/fs/9p/vfs_super.c +++ b/fs/9p/vfs_super.c @@ -172,7 +172,7 @@ static struct dentry *v9fs_mount(struct file_system_type *fs_type, int flags, goto release_sb; } d_inode(root)->i_ino = v9fs_qid2ino(&st->qid); - v9fs_stat2inode_dotl(st, d_inode(root)); + v9fs_stat2inode_dotl(st, d_inode(root), 0); kfree(st); } else { struct p9_wstat *st = NULL; @@ -183,7 +183,7 @@ static struct dentry *v9fs_mount(struct file_system_type *fs_type, int flags, } d_inode(root)->i_ino = v9fs_qid2ino(&st->qid); - v9fs_stat2inode(st, d_inode(root), sb); + v9fs_stat2inode(st, d_inode(root), sb, 0); p9stat_free(st); kfree(st); diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c index e92a2fee3c577..13c1288b04a73 100644 --- a/fs/cifs/cifs_debug.c +++ b/fs/cifs/cifs_debug.c @@ -115,7 +115,12 @@ static void cifs_debug_tcon(struct seq_file *m, struct cifs_tcon *tcon) seq_puts(m, " type: CDROM "); else seq_printf(m, " type: %d ", dev_type); - if (tcon->seal) + + seq_printf(m, "Serial Number: 0x%x", tcon->vol_serial_number); + + if ((tcon->seal) || + (tcon->ses->session_flags & SMB2_SESSION_FLAG_ENCRYPT_DATA) || + (tcon->share_flags & SHI1005_FLAGS_ENCRYPT_DATA)) seq_printf(m, " Encrypted"); if (tcon->nocase) seq_printf(m, " nocase"); @@ -371,6 +376,10 @@ skip_rdma: atomic_read(&server->in_send), atomic_read(&server->num_waiters)); #endif + if (ses->session_flags & SMB2_SESSION_FLAG_ENCRYPT_DATA) + seq_puts(m, " encrypted"); + if (ses->sign) + seq_puts(m, " signed"); seq_puts(m, "\n\tShares:"); j = 0; diff --git a/fs/cifs/cifs_ioctl.h b/fs/cifs/cifs_ioctl.h index d8bce2f862de8..086ddc5108af7 100644 --- a/fs/cifs/cifs_ioctl.h +++ b/fs/cifs/cifs_ioctl.h @@ -43,6 +43,9 @@ struct smb_snapshot_array { /* snapshots[]; */ } __packed; +/* query_info flags */ +#define PASSTHRU_QUERY_INFO 0x00000000 +#define PASSTHRU_FSCTL 0x00000001 struct smb_query_info { __u32 info_type; __u32 file_info_class; diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index f293e052e351d..38feae812b470 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -479,6 +479,14 @@ struct smb_version_operations { struct cifs_tcon *tcon, __le16 *path, int is_dir, unsigned long p); + /* make unix special files (block, char, fifo, socket) */ + int (*make_node)(unsigned int xid, + struct inode *inode, + struct dentry *dentry, + struct cifs_tcon *tcon, + char *full_path, + umode_t mode, + dev_t device_number); }; struct smb_version_values { @@ -735,13 +743,13 @@ in_flight(struct TCP_Server_Info *server) } static inline bool -has_credits(struct TCP_Server_Info *server, int *credits) +has_credits(struct TCP_Server_Info *server, int *credits, int num_credits) { int num; spin_lock(&server->req_lock); num = *credits; spin_unlock(&server->req_lock); - return num > 0; + return num >= num_credits; } static inline void @@ -962,11 +970,14 @@ cap_unix(struct cifs_ses *ses) struct cached_fid { bool is_valid:1; /* Do we have a useable root fid */ + bool file_all_info_is_valid:1; + struct kref refcount; struct cifs_fid *fid; struct mutex fid_mutex; struct cifs_tcon *tcon; struct work_struct lease_break; + struct smb2_file_all_info file_all_info; }; /* @@ -1735,6 +1746,7 @@ require use of the stronger protocol */ * GlobalMid_Lock protects: * list operations on pending_mid_q and oplockQ * updates to XID counters, multiplex id and SMB sequence numbers + * list operations on global DnotifyReqList * tcp_ses_lock protects: * list operations on tcp and SMB session lists * tcon->open_file_lock protects the list of open files hanging off the tcon diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index b95db2b593cb0..a8e9738db6912 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -1191,10 +1191,6 @@ next_pdu: continue; } - if (server->large_buf) - buf = server->bigbuf; - - server->lstrp = jiffies; for (i = 0; i < num_mids; i++) { diff --git a/fs/cifs/dir.c b/fs/cifs/dir.c index 907e85d65bb4e..f26a48dd2e395 100644 --- a/fs/cifs/dir.c +++ b/fs/cifs/dir.c @@ -621,20 +621,10 @@ int cifs_mknod(struct inode *inode, struct dentry *direntry, umode_t mode, { int rc = -EPERM; unsigned int xid; - int create_options = CREATE_NOT_DIR | CREATE_OPTION_SPECIAL; struct cifs_sb_info *cifs_sb; struct tcon_link *tlink; struct cifs_tcon *tcon; - struct cifs_io_parms io_parms; char *full_path = NULL; - struct inode *newinode = NULL; - __u32 oplock = 0; - struct cifs_fid fid; - struct cifs_open_parms oparms; - FILE_ALL_INFO *buf = NULL; - unsigned int bytes_written; - struct win_dev *pdev; - struct kvec iov[2]; if (!old_valid_dev(device_number)) return -EINVAL; @@ -654,103 +644,12 @@ int cifs_mknod(struct inode *inode, struct dentry *direntry, umode_t mode, goto mknod_out; } - if (tcon->unix_ext) { - struct cifs_unix_set_info_args args = { - .mode = mode & ~current_umask(), - .ctime = NO_CHANGE_64, - .atime = NO_CHANGE_64, - .mtime = NO_CHANGE_64, - .device = device_number, - }; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID) { - args.uid = current_fsuid(); - args.gid = current_fsgid(); - } else { - args.uid = INVALID_UID; /* no change */ - args.gid = INVALID_GID; /* no change */ - } - rc = CIFSSMBUnixSetPathInfo(xid, tcon, full_path, &args, - cifs_sb->local_nls, - cifs_remap(cifs_sb)); - if (rc) - goto mknod_out; - - rc = cifs_get_inode_info_unix(&newinode, full_path, - inode->i_sb, xid); - - if (rc == 0) - d_instantiate(direntry, newinode); - goto mknod_out; - } - - if (!S_ISCHR(mode) && !S_ISBLK(mode)) - goto mknod_out; - - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL)) - goto mknod_out; - - - cifs_dbg(FYI, "sfu compat create special file\n"); - - buf = kmalloc(sizeof(FILE_ALL_INFO), GFP_KERNEL); - if (buf == NULL) { - rc = -ENOMEM; - goto mknod_out; - } - - if (backup_cred(cifs_sb)) - create_options |= CREATE_OPEN_BACKUP_INTENT; - - oparms.tcon = tcon; - oparms.cifs_sb = cifs_sb; - oparms.desired_access = GENERIC_WRITE; - oparms.create_options = create_options; - oparms.disposition = FILE_CREATE; - oparms.path = full_path; - oparms.fid = &fid; - oparms.reconnect = false; - - if (tcon->ses->server->oplocks) - oplock = REQ_OPLOCK; - else - oplock = 0; - rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, buf); - if (rc) - goto mknod_out; - - /* - * BB Do not bother to decode buf since no local inode yet to put - * timestamps in, but we can reuse it safely. - */ - - pdev = (struct win_dev *)buf; - io_parms.pid = current->tgid; - io_parms.tcon = tcon; - io_parms.offset = 0; - io_parms.length = sizeof(struct win_dev); - iov[1].iov_base = buf; - iov[1].iov_len = sizeof(struct win_dev); - if (S_ISCHR(mode)) { - memcpy(pdev->type, "IntxCHR", 8); - pdev->major = cpu_to_le64(MAJOR(device_number)); - pdev->minor = cpu_to_le64(MINOR(device_number)); - rc = tcon->ses->server->ops->sync_write(xid, &fid, &io_parms, - &bytes_written, iov, 1); - } else if (S_ISBLK(mode)) { - memcpy(pdev->type, "IntxBLK", 8); - pdev->major = cpu_to_le64(MAJOR(device_number)); - pdev->minor = cpu_to_le64(MINOR(device_number)); - rc = tcon->ses->server->ops->sync_write(xid, &fid, &io_parms, - &bytes_written, iov, 1); - } - tcon->ses->server->ops->close(xid, tcon, &fid); - d_drop(direntry); - - /* FIXME: add code here to set EAs */ + rc = tcon->ses->server->ops->make_node(xid, inode, direntry, tcon, + full_path, mode, + device_number); mknod_out: kfree(full_path); - kfree(buf); free_xid(xid); cifs_put_tlink(tlink); return rc; diff --git a/fs/cifs/file.c b/fs/cifs/file.c index 4c144c1f50eb5..2a6d20c0ce028 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c @@ -1645,8 +1645,20 @@ cifs_setlk(struct file *file, struct file_lock *flock, __u32 type, rc = server->ops->mand_unlock_range(cfile, flock, xid); out: - if (flock->fl_flags & FL_POSIX && !rc) + if (flock->fl_flags & FL_POSIX) { + /* + * If this is a request to remove all locks because we + * are closing the file, it doesn't matter if the + * unlocking failed as both cifs.ko and the SMB server + * remove the lock on file close + */ + if (rc) { + cifs_dbg(VFS, "%s failed rc=%d\n", __func__, rc); + if (!(flock->fl_flags & FL_CLOSE)) + return rc; + } rc = locks_lock_file_wait(file, flock); + } return rc; } diff --git a/fs/cifs/smb1ops.c b/fs/cifs/smb1ops.c index f0ce27c3c6e44..c711f1f39bf2e 100644 --- a/fs/cifs/smb1ops.c +++ b/fs/cifs/smb1ops.c @@ -1027,6 +1027,131 @@ cifs_can_echo(struct TCP_Server_Info *server) return false; } +static int +cifs_make_node(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + char *full_path, umode_t mode, dev_t dev) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct inode *newinode = NULL; + int rc = -EPERM; + int create_options = CREATE_NOT_DIR | CREATE_OPTION_SPECIAL; + FILE_ALL_INFO *buf = NULL; + struct cifs_io_parms io_parms; + __u32 oplock = 0; + struct cifs_fid fid; + struct cifs_open_parms oparms; + unsigned int bytes_written; + struct win_dev *pdev; + struct kvec iov[2]; + + if (tcon->unix_ext) { + /* + * SMB1 Unix Extensions: requires server support but + * works with all special files + */ + struct cifs_unix_set_info_args args = { + .mode = mode & ~current_umask(), + .ctime = NO_CHANGE_64, + .atime = NO_CHANGE_64, + .mtime = NO_CHANGE_64, + .device = dev, + }; + if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID) { + args.uid = current_fsuid(); + args.gid = current_fsgid(); + } else { + args.uid = INVALID_UID; /* no change */ + args.gid = INVALID_GID; /* no change */ + } + rc = CIFSSMBUnixSetPathInfo(xid, tcon, full_path, &args, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + if (rc) + goto out; + + rc = cifs_get_inode_info_unix(&newinode, full_path, + inode->i_sb, xid); + + if (rc == 0) + d_instantiate(dentry, newinode); + goto out; + } + + /* + * SMB1 SFU emulation: should work with all servers, but only + * support block and char device (no socket & fifo) + */ + if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL)) + goto out; + + if (!S_ISCHR(mode) && !S_ISBLK(mode)) + goto out; + + cifs_dbg(FYI, "sfu compat create special file\n"); + + buf = kmalloc(sizeof(FILE_ALL_INFO), GFP_KERNEL); + if (buf == NULL) { + rc = -ENOMEM; + goto out; + } + + if (backup_cred(cifs_sb)) + create_options |= CREATE_OPEN_BACKUP_INTENT; + + oparms.tcon = tcon; + oparms.cifs_sb = cifs_sb; + oparms.desired_access = GENERIC_WRITE; + oparms.create_options = create_options; + oparms.disposition = FILE_CREATE; + oparms.path = full_path; + oparms.fid = &fid; + oparms.reconnect = false; + + if (tcon->ses->server->oplocks) + oplock = REQ_OPLOCK; + else + oplock = 0; + rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, buf); + if (rc) + goto out; + + /* + * BB Do not bother to decode buf since no local inode yet to put + * timestamps in, but we can reuse it safely. + */ + + pdev = (struct win_dev *)buf; + io_parms.pid = current->tgid; + io_parms.tcon = tcon; + io_parms.offset = 0; + io_parms.length = sizeof(struct win_dev); + iov[1].iov_base = buf; + iov[1].iov_len = sizeof(struct win_dev); + if (S_ISCHR(mode)) { + memcpy(pdev->type, "IntxCHR", 8); + pdev->major = cpu_to_le64(MAJOR(dev)); + pdev->minor = cpu_to_le64(MINOR(dev)); + rc = tcon->ses->server->ops->sync_write(xid, &fid, &io_parms, + &bytes_written, iov, 1); + } else if (S_ISBLK(mode)) { + memcpy(pdev->type, "IntxBLK", 8); + pdev->major = cpu_to_le64(MAJOR(dev)); + pdev->minor = cpu_to_le64(MINOR(dev)); + rc = tcon->ses->server->ops->sync_write(xid, &fid, &io_parms, + &bytes_written, iov, 1); + } + tcon->ses->server->ops->close(xid, tcon, &fid); + d_drop(dentry); + + /* FIXME: add code here to set EAs */ +out: + kfree(buf); + return rc; +} + + + struct smb_version_operations smb1_operations = { .send_cancel = send_nt_cancel, .compare_fids = cifs_compare_fids, @@ -1110,6 +1235,7 @@ struct smb_version_operations smb1_operations = { .get_acl_by_fid = get_cifs_acl_by_fid, .set_acl = set_cifs_acl, #endif /* CIFS_ACL */ + .make_node = cifs_make_node, }; struct smb_version_values smb1_values = { diff --git a/fs/cifs/smb2inode.c b/fs/cifs/smb2inode.c index 01a76bccdb8df..278405d26c47e 100644 --- a/fs/cifs/smb2inode.c +++ b/fs/cifs/smb2inode.c @@ -37,6 +37,16 @@ #include "smb2pdu.h" #include "smb2proto.h" +static void +free_set_inf_compound(struct smb_rqst *rqst) +{ + if (rqst[1].rq_iov) + SMB2_set_info_free(&rqst[1]); + if (rqst[2].rq_iov) + SMB2_close_free(&rqst[2]); +} + + static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_sb_info *cifs_sb, const char *full_path, @@ -112,14 +122,18 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, PATH_MAX * 2, 0, NULL); smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); + trace_smb3_query_info_compound_enter(xid, ses->Suid, tcon->tid, + full_path); break; case SMB2_OP_DELETE: + trace_smb3_delete_enter(xid, ses->Suid, tcon->tid, full_path); break; case SMB2_OP_MKDIR: /* * Directories are created through parameters in the * SMB2_open() call. */ + trace_smb3_mkdir_enter(xid, ses->Suid, tcon->tid, full_path); break; case SMB2_OP_RMDIR: memset(&si_iov, 0, sizeof(si_iov)); @@ -135,6 +149,7 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, SMB2_O_INFO_FILE, 0, data, size); smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); + trace_smb3_rmdir_enter(xid, ses->Suid, tcon->tid, full_path); break; case SMB2_OP_SET_EOF: memset(&si_iov, 0, sizeof(si_iov)); @@ -150,6 +165,7 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, SMB2_O_INFO_FILE, 0, data, size); smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); + trace_smb3_set_eof_enter(xid, ses->Suid, tcon->tid, full_path); break; case SMB2_OP_SET_INFO: memset(&si_iov, 0, sizeof(si_iov)); @@ -166,6 +182,8 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, SMB2_O_INFO_FILE, 0, data, size); smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); + trace_smb3_set_info_compound_enter(xid, ses->Suid, tcon->tid, + full_path); break; case SMB2_OP_RENAME: memset(&si_iov, 0, sizeof(si_iov)); @@ -190,6 +208,7 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, SMB2_O_INFO_FILE, 0, data, size); smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); + trace_smb3_rename_enter(xid, ses->Suid, tcon->tid, full_path); break; case SMB2_OP_HARDLINK: memset(&si_iov, 0, sizeof(si_iov)); @@ -214,6 +233,7 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, SMB2_O_INFO_FILE, 0, data, size); smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); + trace_smb3_hardlink_enter(xid, ses->Suid, tcon->tid, full_path); break; default: cifs_dbg(VFS, "Invalid command\n"); @@ -252,21 +272,65 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, SMB2_query_info_free(&rqst[1]); if (rqst[2].rq_iov) SMB2_close_free(&rqst[2]); + if (rc) + trace_smb3_query_info_compound_err(xid, ses->Suid, + tcon->tid, rc); + else + trace_smb3_query_info_compound_done(xid, ses->Suid, + tcon->tid); break; case SMB2_OP_DELETE: + if (rc) + trace_smb3_delete_err(xid, ses->Suid, tcon->tid, rc); + else + trace_smb3_delete_done(xid, ses->Suid, tcon->tid); + if (rqst[1].rq_iov) + SMB2_close_free(&rqst[1]); + break; case SMB2_OP_MKDIR: + if (rc) + trace_smb3_mkdir_err(xid, ses->Suid, tcon->tid, rc); + else + trace_smb3_mkdir_done(xid, ses->Suid, tcon->tid); if (rqst[1].rq_iov) SMB2_close_free(&rqst[1]); break; case SMB2_OP_HARDLINK: + if (rc) + trace_smb3_hardlink_err(xid, ses->Suid, tcon->tid, rc); + else + trace_smb3_hardlink_done(xid, ses->Suid, tcon->tid); + free_set_inf_compound(rqst); + break; case SMB2_OP_RENAME: + if (rc) + trace_smb3_rename_err(xid, ses->Suid, tcon->tid, rc); + else + trace_smb3_rename_done(xid, ses->Suid, tcon->tid); + free_set_inf_compound(rqst); + break; case SMB2_OP_RMDIR: + if (rc) + trace_smb3_rmdir_err(xid, ses->Suid, tcon->tid, rc); + else + trace_smb3_rmdir_done(xid, ses->Suid, tcon->tid); + free_set_inf_compound(rqst); + break; case SMB2_OP_SET_EOF: + if (rc) + trace_smb3_set_eof_err(xid, ses->Suid, tcon->tid, rc); + else + trace_smb3_set_eof_done(xid, ses->Suid, tcon->tid); + free_set_inf_compound(rqst); + break; case SMB2_OP_SET_INFO: - if (rqst[1].rq_iov) - SMB2_set_info_free(&rqst[1]); - if (rqst[2].rq_iov) - SMB2_close_free(&rqst[2]); + if (rc) + trace_smb3_set_info_compound_err(xid, ses->Suid, + tcon->tid, rc); + else + trace_smb3_set_info_compound_done(xid, ses->Suid, + tcon->tid); + free_set_inf_compound(rqst); break; } free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); @@ -309,12 +373,17 @@ smb2_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, rc = open_shroot(xid, tcon, &fid); if (rc) goto out; - rc = SMB2_query_info(xid, tcon, fid.persistent_fid, - fid.volatile_fid, smb2_data); + + if (tcon->crfid.file_all_info_is_valid) { + move_smb2_info_to_cifs(data, + &tcon->crfid.file_all_info); + } else { + rc = SMB2_query_info(xid, tcon, fid.persistent_fid, + fid.volatile_fid, smb2_data); + if (!rc) + move_smb2_info_to_cifs(data, smb2_data); + } close_shroot(&tcon->crfid); - if (rc) - goto out; - move_smb2_info_to_cifs(data, smb2_data); goto out; } diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index 085e91436da7e..1022a3771e140 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -185,7 +185,7 @@ smb2_wait_mtu_credits(struct TCP_Server_Info *server, unsigned int size, spin_unlock(&server->req_lock); cifs_num_waiters_inc(server); rc = wait_event_killable(server->request_q, - has_credits(server, &server->credits)); + has_credits(server, &server->credits, 1)); cifs_num_waiters_dec(server); if (rc) return rc; @@ -619,6 +619,7 @@ smb2_close_cached_fid(struct kref *ref) SMB2_close(0, cfid->tcon, cfid->fid->persistent_fid, cfid->fid->volatile_fid); cfid->is_valid = false; + cfid->file_all_info_is_valid = false; } } @@ -643,9 +644,18 @@ smb2_cached_lease_break(struct work_struct *work) */ int open_shroot(unsigned int xid, struct cifs_tcon *tcon, struct cifs_fid *pfid) { - struct cifs_open_parms oparams; - int rc; - __le16 srch_path = 0; /* Null - since an open of top of share */ + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server = ses->server; + struct cifs_open_parms oparms; + struct smb2_create_rsp *o_rsp = NULL; + struct smb2_query_info_rsp *qi_rsp = NULL; + int resp_buftype[2]; + struct smb_rqst rqst[2]; + struct kvec rsp_iov[2]; + struct kvec open_iov[SMB2_CREATE_IOV_SIZE]; + struct kvec qi_iov[1]; + int rc, flags = 0; + __le16 utf16_path = 0; /* Null - since an open of top of share */ u8 oplock = SMB2_OPLOCK_LEVEL_II; mutex_lock(&tcon->crfid.fid_mutex); @@ -657,22 +667,89 @@ int open_shroot(unsigned int xid, struct cifs_tcon *tcon, struct cifs_fid *pfid) return 0; } - oparams.tcon = tcon; - oparams.create_options = 0; - oparams.desired_access = FILE_READ_ATTRIBUTES; - oparams.disposition = FILE_OPEN; - oparams.fid = pfid; - oparams.reconnect = false; - - rc = SMB2_open(xid, &oparams, &srch_path, &oplock, NULL, NULL, NULL); - if (rc == 0) { - memcpy(tcon->crfid.fid, pfid, sizeof(struct cifs_fid)); - tcon->crfid.tcon = tcon; - tcon->crfid.is_valid = true; - kref_init(&tcon->crfid.refcount); - kref_get(&tcon->crfid.refcount); - } + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(rqst, 0, sizeof(rqst)); + resp_buftype[0] = resp_buftype[1] = CIFS_NO_BUFFER; + memset(rsp_iov, 0, sizeof(rsp_iov)); + + /* Open */ + memset(&open_iov, 0, sizeof(open_iov)); + rqst[0].rq_iov = open_iov; + rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE; + + oparms.tcon = tcon; + oparms.create_options = 0; + oparms.desired_access = FILE_READ_ATTRIBUTES; + oparms.disposition = FILE_OPEN; + oparms.fid = pfid; + oparms.reconnect = false; + + rc = SMB2_open_init(tcon, &rqst[0], &oplock, &oparms, &utf16_path); + if (rc) + goto oshr_exit; + smb2_set_next_command(tcon, &rqst[0]); + + memset(&qi_iov, 0, sizeof(qi_iov)); + rqst[1].rq_iov = qi_iov; + rqst[1].rq_nvec = 1; + + rc = SMB2_query_info_init(tcon, &rqst[1], COMPOUND_FID, + COMPOUND_FID, FILE_ALL_INFORMATION, + SMB2_O_INFO_FILE, 0, + sizeof(struct smb2_file_all_info) + + PATH_MAX * 2, 0, NULL); + if (rc) + goto oshr_exit; + + smb2_set_related(&rqst[1]); + + rc = compound_send_recv(xid, ses, flags, 2, rqst, + resp_buftype, rsp_iov); + if (rc) + goto oshr_exit; + + o_rsp = (struct smb2_create_rsp *)rsp_iov[0].iov_base; + oparms.fid->persistent_fid = o_rsp->PersistentFileId; + oparms.fid->volatile_fid = o_rsp->VolatileFileId; +#ifdef CONFIG_CIFS_DEBUG2 + oparms.fid->mid = le64_to_cpu(o_rsp->sync_hdr.MessageId); +#endif /* CIFS_DEBUG2 */ + + if (o_rsp->OplockLevel == SMB2_OPLOCK_LEVEL_LEASE) + oplock = smb2_parse_lease_state(server, o_rsp, + &oparms.fid->epoch, + oparms.fid->lease_key); + else + goto oshr_exit; + + + memcpy(tcon->crfid.fid, pfid, sizeof(struct cifs_fid)); + tcon->crfid.tcon = tcon; + tcon->crfid.is_valid = true; + kref_init(&tcon->crfid.refcount); + kref_get(&tcon->crfid.refcount); + + + qi_rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base; + if (le32_to_cpu(qi_rsp->OutputBufferLength) < sizeof(struct smb2_file_all_info)) + goto oshr_exit; + rc = smb2_validate_and_copy_iov( + le16_to_cpu(qi_rsp->OutputBufferOffset), + sizeof(struct smb2_file_all_info), + &rsp_iov[1], sizeof(struct smb2_file_all_info), + (char *)&tcon->crfid.file_all_info); + if (rc) + goto oshr_exit; + tcon->crfid.file_all_info_is_valid = 1; + + oshr_exit: mutex_unlock(&tcon->crfid.fid_mutex); + SMB2_open_free(&rqst[0]); + SMB2_query_info_free(&rqst[1]); + free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); + free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); return rc; } @@ -1253,7 +1330,8 @@ smb2_ioctl_query_info(const unsigned int xid, struct smb_query_info __user *pqi; int rc = 0; int flags = 0; - struct smb2_query_info_rsp *rsp = NULL; + struct smb2_query_info_rsp *qi_rsp = NULL; + struct smb2_ioctl_rsp *io_rsp = NULL; void *buffer = NULL; struct smb_rqst rqst[3]; int resp_buftype[3]; @@ -1263,6 +1341,7 @@ smb2_ioctl_query_info(const unsigned int xid, u8 oplock = SMB2_OPLOCK_LEVEL_NONE; struct cifs_fid fid; struct kvec qi_iov[1]; + struct kvec io_iov[SMB2_IOCTL_IOV_SIZE]; struct kvec close_iov[1]; memset(rqst, 0, sizeof(rqst)); @@ -1313,15 +1392,35 @@ smb2_ioctl_query_info(const unsigned int xid, smb2_set_next_command(tcon, &rqst[0]); /* Query */ - memset(&qi_iov, 0, sizeof(qi_iov)); - rqst[1].rq_iov = qi_iov; - rqst[1].rq_nvec = 1; - - rc = SMB2_query_info_init(tcon, &rqst[1], COMPOUND_FID, COMPOUND_FID, - qi.file_info_class, qi.info_type, - qi.additional_information, + if (qi.flags & PASSTHRU_FSCTL) { + /* Can eventually relax perm check since server enforces too */ + if (!capable(CAP_SYS_ADMIN)) + rc = -EPERM; + else { + memset(&io_iov, 0, sizeof(io_iov)); + rqst[1].rq_iov = io_iov; + rqst[1].rq_nvec = SMB2_IOCTL_IOV_SIZE; + + rc = SMB2_ioctl_init(tcon, &rqst[1], + COMPOUND_FID, COMPOUND_FID, + qi.info_type, true, NULL, + 0); + } + } else if (qi.flags == PASSTHRU_QUERY_INFO) { + memset(&qi_iov, 0, sizeof(qi_iov)); + rqst[1].rq_iov = qi_iov; + rqst[1].rq_nvec = 1; + + rc = SMB2_query_info_init(tcon, &rqst[1], COMPOUND_FID, + COMPOUND_FID, qi.file_info_class, + qi.info_type, qi.additional_information, qi.input_buffer_length, qi.output_buffer_length, buffer); + } else { /* unknown flags */ + cifs_dbg(VFS, "invalid passthru query flags: 0x%x\n", qi.flags); + rc = -EINVAL; + } + if (rc) goto iqinf_exit; smb2_set_next_command(tcon, &rqst[1]); @@ -1341,24 +1440,44 @@ smb2_ioctl_query_info(const unsigned int xid, resp_buftype, rsp_iov); if (rc) goto iqinf_exit; - pqi = (struct smb_query_info __user *)arg; - rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base; - if (le32_to_cpu(rsp->OutputBufferLength) < qi.input_buffer_length) - qi.input_buffer_length = le32_to_cpu(rsp->OutputBufferLength); - if (copy_to_user(&pqi->input_buffer_length, &qi.input_buffer_length, - sizeof(qi.input_buffer_length))) { - rc = -EFAULT; - goto iqinf_exit; - } - if (copy_to_user(pqi + 1, rsp->Buffer, qi.input_buffer_length)) { - rc = -EFAULT; - goto iqinf_exit; + if (qi.flags & PASSTHRU_FSCTL) { + pqi = (struct smb_query_info __user *)arg; + io_rsp = (struct smb2_ioctl_rsp *)rsp_iov[1].iov_base; + if (le32_to_cpu(io_rsp->OutputCount) < qi.input_buffer_length) + qi.input_buffer_length = le32_to_cpu(io_rsp->OutputCount); + if (copy_to_user(&pqi->input_buffer_length, &qi.input_buffer_length, + sizeof(qi.input_buffer_length))) { + rc = -EFAULT; + goto iqinf_exit; + } + if (copy_to_user(pqi + 1, &io_rsp[1], qi.input_buffer_length)) { + rc = -EFAULT; + goto iqinf_exit; + } + } else { + pqi = (struct smb_query_info __user *)arg; + qi_rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base; + if (le32_to_cpu(qi_rsp->OutputBufferLength) < qi.input_buffer_length) + qi.input_buffer_length = le32_to_cpu(qi_rsp->OutputBufferLength); + if (copy_to_user(&pqi->input_buffer_length, &qi.input_buffer_length, + sizeof(qi.input_buffer_length))) { + rc = -EFAULT; + goto iqinf_exit; + } + if (copy_to_user(pqi + 1, qi_rsp->Buffer, qi.input_buffer_length)) { + rc = -EFAULT; + goto iqinf_exit; + } } iqinf_exit: kfree(buffer); SMB2_open_free(&rqst[0]); - SMB2_query_info_free(&rqst[1]); + if (qi.flags & PASSTHRU_FSCTL) + SMB2_ioctl_free(&rqst[1]); + else + SMB2_query_info_free(&rqst[1]); + SMB2_close_free(&rqst[2]); free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); @@ -2472,22 +2591,38 @@ get_smb2_acl(struct cifs_sb_info *cifs_sb, static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, loff_t offset, loff_t len, bool keep_size) { + struct cifs_ses *ses = tcon->ses; struct inode *inode; struct cifsInodeInfo *cifsi; struct cifsFileInfo *cfile = file->private_data; struct file_zero_data_information fsctl_buf; + struct smb_rqst rqst[2]; + int resp_buftype[2]; + struct kvec rsp_iov[2]; + struct kvec io_iov[SMB2_IOCTL_IOV_SIZE]; + struct kvec si_iov[1]; + unsigned int size[1]; + void *data[1]; long rc; unsigned int xid; + int num = 0, flags = 0; + __le64 eof; xid = get_xid(); inode = d_inode(cfile->dentry); cifsi = CIFS_I(inode); + trace_smb3_zero_enter(xid, cfile->fid.persistent_fid, tcon->tid, + ses->Suid, offset, len); + + /* if file not oplocked can't be sure whether asking to extend size */ if (!CIFS_CACHE_READ(cifsi)) if (keep_size == false) { rc = -EOPNOTSUPP; + trace_smb3_zero_err(xid, cfile->fid.persistent_fid, + tcon->tid, ses->Suid, offset, len, rc); free_xid(xid); return rc; } @@ -2498,33 +2633,73 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, */ if (!(cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE)) { rc = -EOPNOTSUPP; + trace_smb3_zero_err(xid, cfile->fid.persistent_fid, tcon->tid, + ses->Suid, offset, len, rc); free_xid(xid); return rc; } - /* - * need to make sure we are not asked to extend the file since the SMB3 - * fsctl does not change the file size. In the future we could change - * this to zero the first part of the range then set the file size - * which for a non sparse file would zero the newly extended range - */ - if (keep_size == false) - if (i_size_read(inode) < offset + len) { - rc = -EOPNOTSUPP; - free_xid(xid); - return rc; - } - cifs_dbg(FYI, "offset %lld len %lld", offset, len); fsctl_buf.FileOffset = cpu_to_le64(offset); fsctl_buf.BeyondFinalZero = cpu_to_le64(offset + len); - rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, - cfile->fid.volatile_fid, FSCTL_SET_ZERO_DATA, - true /* is_fctl */, (char *)&fsctl_buf, - sizeof(struct file_zero_data_information), NULL, NULL); + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(rqst, 0, sizeof(rqst)); + resp_buftype[0] = resp_buftype[1] = CIFS_NO_BUFFER; + memset(rsp_iov, 0, sizeof(rsp_iov)); + + + memset(&io_iov, 0, sizeof(io_iov)); + rqst[num].rq_iov = io_iov; + rqst[num].rq_nvec = SMB2_IOCTL_IOV_SIZE; + rc = SMB2_ioctl_init(tcon, &rqst[num++], cfile->fid.persistent_fid, + cfile->fid.volatile_fid, FSCTL_SET_ZERO_DATA, + true /* is_fctl */, (char *)&fsctl_buf, + sizeof(struct file_zero_data_information)); + if (rc) + goto zero_range_exit; + + /* + * do we also need to change the size of the file? + */ + if (keep_size == false && i_size_read(inode) < offset + len) { + smb2_set_next_command(tcon, &rqst[0]); + + memset(&si_iov, 0, sizeof(si_iov)); + rqst[num].rq_iov = si_iov; + rqst[num].rq_nvec = 1; + + eof = cpu_to_le64(offset + len); + size[0] = 8; /* sizeof __le64 */ + data[0] = &eof; + + rc = SMB2_set_info_init(tcon, &rqst[num++], + cfile->fid.persistent_fid, + cfile->fid.persistent_fid, + current->tgid, + FILE_END_OF_FILE_INFORMATION, + SMB2_O_INFO_FILE, 0, data, size); + smb2_set_related(&rqst[1]); + } + + rc = compound_send_recv(xid, ses, flags, num, rqst, + resp_buftype, rsp_iov); + + zero_range_exit: + SMB2_ioctl_free(&rqst[0]); + SMB2_set_info_free(&rqst[1]); + free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); + free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); free_xid(xid); + if (rc) + trace_smb3_zero_err(xid, cfile->fid.persistent_fid, tcon->tid, + ses->Suid, offset, len, rc); + else + trace_smb3_zero_done(xid, cfile->fid.persistent_fid, tcon->tid, + ses->Suid, offset, len); return rc; } @@ -2573,15 +2748,20 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, struct cifsFileInfo *cfile = file->private_data; long rc = -EOPNOTSUPP; unsigned int xid; + __le64 eof; xid = get_xid(); inode = d_inode(cfile->dentry); cifsi = CIFS_I(inode); + trace_smb3_falloc_enter(xid, cfile->fid.persistent_fid, tcon->tid, + tcon->ses->Suid, off, len); /* if file not oplocked can't be sure whether asking to extend size */ if (!CIFS_CACHE_READ(cifsi)) if (keep_size == false) { + trace_smb3_falloc_err(xid, cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, off, len, rc); free_xid(xid); return rc; } @@ -2601,6 +2781,12 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, /* BB: in future add else clause to extend file */ else rc = -EOPNOTSUPP; + if (rc) + trace_smb3_falloc_err(xid, cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, off, len, rc); + else + trace_smb3_falloc_done(xid, cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, off, len); free_xid(xid); return rc; } @@ -2616,14 +2802,31 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, */ if ((off > 8192) || (off + len + 8192 < i_size_read(inode))) { rc = -EOPNOTSUPP; + trace_smb3_falloc_err(xid, cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, off, len, rc); free_xid(xid); return rc; } - rc = smb2_set_sparse(xid, tcon, cfile, inode, false); + smb2_set_sparse(xid, tcon, cfile, inode, false); + rc = 0; + } else { + smb2_set_sparse(xid, tcon, cfile, inode, false); + rc = 0; + if (i_size_read(inode) < off + len) { + eof = cpu_to_le64(off + len); + rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, cfile->pid, + &eof); + } } - /* BB: else ... in future add code to extend file and set sparse */ + if (rc) + trace_smb3_falloc_err(xid, cfile->fid.persistent_fid, tcon->tid, + tcon->ses->Suid, off, len, rc); + else + trace_smb3_falloc_done(xid, cfile->fid.persistent_fid, tcon->tid, + tcon->ses->Suid, off, len); free_xid(xid); return rc; @@ -3604,6 +3807,104 @@ smb2_next_header(char *buf) return le32_to_cpu(hdr->NextCommand); } +static int +smb2_make_node(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + char *full_path, umode_t mode, dev_t dev) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + int rc = -EPERM; + int create_options = CREATE_NOT_DIR | CREATE_OPTION_SPECIAL; + FILE_ALL_INFO *buf = NULL; + struct cifs_io_parms io_parms; + __u32 oplock = 0; + struct cifs_fid fid; + struct cifs_open_parms oparms; + unsigned int bytes_written; + struct win_dev *pdev; + struct kvec iov[2]; + + /* + * Check if mounted with mount parm 'sfu' mount parm. + * SFU emulation should work with all servers, but only + * supports block and char device (no socket & fifo), + * and was used by default in earlier versions of Windows + */ + if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL)) + goto out; + + /* + * TODO: Add ability to create instead via reparse point. Windows (e.g. + * their current NFS server) uses this approach to expose special files + * over SMB2/SMB3 and Samba will do this with SMB3.1.1 POSIX Extensions + */ + + if (!S_ISCHR(mode) && !S_ISBLK(mode)) + goto out; + + cifs_dbg(FYI, "sfu compat create special file\n"); + + buf = kmalloc(sizeof(FILE_ALL_INFO), GFP_KERNEL); + if (buf == NULL) { + rc = -ENOMEM; + goto out; + } + + if (backup_cred(cifs_sb)) + create_options |= CREATE_OPEN_BACKUP_INTENT; + + oparms.tcon = tcon; + oparms.cifs_sb = cifs_sb; + oparms.desired_access = GENERIC_WRITE; + oparms.create_options = create_options; + oparms.disposition = FILE_CREATE; + oparms.path = full_path; + oparms.fid = &fid; + oparms.reconnect = false; + + if (tcon->ses->server->oplocks) + oplock = REQ_OPLOCK; + else + oplock = 0; + rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, buf); + if (rc) + goto out; + + /* + * BB Do not bother to decode buf since no local inode yet to put + * timestamps in, but we can reuse it safely. + */ + + pdev = (struct win_dev *)buf; + io_parms.pid = current->tgid; + io_parms.tcon = tcon; + io_parms.offset = 0; + io_parms.length = sizeof(struct win_dev); + iov[1].iov_base = buf; + iov[1].iov_len = sizeof(struct win_dev); + if (S_ISCHR(mode)) { + memcpy(pdev->type, "IntxCHR", 8); + pdev->major = cpu_to_le64(MAJOR(dev)); + pdev->minor = cpu_to_le64(MINOR(dev)); + rc = tcon->ses->server->ops->sync_write(xid, &fid, &io_parms, + &bytes_written, iov, 1); + } else if (S_ISBLK(mode)) { + memcpy(pdev->type, "IntxBLK", 8); + pdev->major = cpu_to_le64(MAJOR(dev)); + pdev->minor = cpu_to_le64(MINOR(dev)); + rc = tcon->ses->server->ops->sync_write(xid, &fid, &io_parms, + &bytes_written, iov, 1); + } + tcon->ses->server->ops->close(xid, tcon, &fid); + d_drop(dentry); + + /* FIXME: add code here to set EAs */ +out: + kfree(buf); + return rc; +} + + struct smb_version_operations smb20_operations = { .compare_fids = smb2_compare_fids, .setup_request = smb2_setup_request, @@ -3698,6 +3999,7 @@ struct smb_version_operations smb20_operations = { #endif /* CIFS_ACL */ .next_header = smb2_next_header, .ioctl_query_info = smb2_ioctl_query_info, + .make_node = smb2_make_node, }; struct smb_version_operations smb21_operations = { @@ -3796,6 +4098,7 @@ struct smb_version_operations smb21_operations = { #endif /* CIFS_ACL */ .next_header = smb2_next_header, .ioctl_query_info = smb2_ioctl_query_info, + .make_node = smb2_make_node, }; struct smb_version_operations smb30_operations = { @@ -3903,6 +4206,7 @@ struct smb_version_operations smb30_operations = { #endif /* CIFS_ACL */ .next_header = smb2_next_header, .ioctl_query_info = smb2_ioctl_query_info, + .make_node = smb2_make_node, }; struct smb_version_operations smb311_operations = { @@ -4011,6 +4315,7 @@ struct smb_version_operations smb311_operations = { #endif /* CIFS_ACL */ .next_header = smb2_next_header, .ioctl_query_info = smb2_ioctl_query_info, + .make_node = smb2_make_node, }; struct smb_version_values smb20_values = { diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c index 60fbe306f6043..c399e09b76e62 100644 --- a/fs/cifs/smb2pdu.c +++ b/fs/cifs/smb2pdu.c @@ -1797,9 +1797,10 @@ create_reconnect_durable_buf(struct cifs_fid *fid) return buf; } -static __u8 -parse_lease_state(struct TCP_Server_Info *server, struct smb2_create_rsp *rsp, - unsigned int *epoch, char *lease_key) +__u8 +smb2_parse_lease_state(struct TCP_Server_Info *server, + struct smb2_create_rsp *rsp, + unsigned int *epoch, char *lease_key) { char *data_offset; struct create_context *cc; @@ -2456,8 +2457,9 @@ SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path, } if (rsp->OplockLevel == SMB2_OPLOCK_LEVEL_LEASE) - *oplock = parse_lease_state(server, rsp, &oparms->fid->epoch, - oparms->fid->lease_key); + *oplock = smb2_parse_lease_state(server, rsp, + &oparms->fid->epoch, + oparms->fid->lease_key); else *oplock = rsp->OplockLevel; creat_exit: @@ -2466,65 +2468,46 @@ creat_exit: return rc; } -/* - * SMB2 IOCTL is used for both IOCTLs and FSCTLs - */ int -SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, - u64 volatile_fid, u32 opcode, bool is_fsctl, - char *in_data, u32 indatalen, - char **out_data, u32 *plen /* returned data len */) +SMB2_ioctl_init(struct cifs_tcon *tcon, struct smb_rqst *rqst, + u64 persistent_fid, u64 volatile_fid, u32 opcode, + bool is_fsctl, char *in_data, u32 indatalen) { - struct smb_rqst rqst; struct smb2_ioctl_req *req; - struct smb2_ioctl_rsp *rsp; - struct cifs_ses *ses; - struct kvec iov[2]; - struct kvec rsp_iov; - int resp_buftype; - int n_iov; - int rc = 0; - int flags = 0; + struct kvec *iov = rqst->rq_iov; unsigned int total_len; - - cifs_dbg(FYI, "SMB2 IOCTL\n"); - - if (out_data != NULL) - *out_data = NULL; - - /* zero out returned data len, in case of error */ - if (plen) - *plen = 0; - - if (tcon) - ses = tcon->ses; - else - return -EIO; - - if (!ses || !(ses->server)) - return -EIO; + int rc; rc = smb2_plain_req_init(SMB2_IOCTL, tcon, (void **) &req, &total_len); if (rc) return rc; - if (smb3_encryption_required(tcon)) - flags |= CIFS_TRANSFORM_REQ; - req->CtlCode = cpu_to_le32(opcode); req->PersistentFileId = persistent_fid; req->VolatileFileId = volatile_fid; + iov[0].iov_base = (char *)req; + /* + * If no input data, the size of ioctl struct in + * protocol spec still includes a 1 byte data buffer, + * but if input data passed to ioctl, we do not + * want to double count this, so we do not send + * the dummy one byte of data in iovec[0] if sending + * input data (in iovec[1]). + */ if (indatalen) { req->InputCount = cpu_to_le32(indatalen); /* do not set InputOffset if no input data */ req->InputOffset = cpu_to_le32(offsetof(struct smb2_ioctl_req, Buffer)); + rqst->rq_nvec = 2; + iov[0].iov_len = total_len - 1; iov[1].iov_base = in_data; iov[1].iov_len = indatalen; - n_iov = 2; - } else - n_iov = 1; + } else { + rqst->rq_nvec = 1; + iov[0].iov_len = total_len; + } req->OutputOffset = 0; req->OutputCount = 0; /* MBZ */ @@ -2546,33 +2529,70 @@ SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, else req->Flags = 0; - iov[0].iov_base = (char *)req; - - /* - * If no input data, the size of ioctl struct in - * protocol spec still includes a 1 byte data buffer, - * but if input data passed to ioctl, we do not - * want to double count this, so we do not send - * the dummy one byte of data in iovec[0] if sending - * input data (in iovec[1]). - */ - - if (indatalen) { - iov[0].iov_len = total_len - 1; - } else - iov[0].iov_len = total_len; - /* validate negotiate request must be signed - see MS-SMB2 3.2.5.5 */ if (opcode == FSCTL_VALIDATE_NEGOTIATE_INFO) req->sync_hdr.Flags |= SMB2_FLAGS_SIGNED; + return 0; +} + +void +SMB2_ioctl_free(struct smb_rqst *rqst) +{ + if (rqst && rqst->rq_iov) + cifs_small_buf_release(rqst->rq_iov[0].iov_base); /* request */ +} + +/* + * SMB2 IOCTL is used for both IOCTLs and FSCTLs + */ +int +SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, + u64 volatile_fid, u32 opcode, bool is_fsctl, + char *in_data, u32 indatalen, + char **out_data, u32 *plen /* returned data len */) +{ + struct smb_rqst rqst; + struct smb2_ioctl_rsp *rsp = NULL; + struct cifs_ses *ses; + struct kvec iov[SMB2_IOCTL_IOV_SIZE]; + struct kvec rsp_iov = {NULL, 0}; + int resp_buftype = CIFS_NO_BUFFER; + int rc = 0; + int flags = 0; + + cifs_dbg(FYI, "SMB2 IOCTL\n"); + + if (out_data != NULL) + *out_data = NULL; + + /* zero out returned data len, in case of error */ + if (plen) + *plen = 0; + + if (tcon) + ses = tcon->ses; + else + return -EIO; + + if (!ses || !(ses->server)) + return -EIO; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + memset(&rqst, 0, sizeof(struct smb_rqst)); + memset(&iov, 0, sizeof(iov)); rqst.rq_iov = iov; - rqst.rq_nvec = n_iov; + rqst.rq_nvec = SMB2_IOCTL_IOV_SIZE; + + rc = SMB2_ioctl_init(tcon, &rqst, persistent_fid, volatile_fid, + opcode, is_fsctl, in_data, indatalen); + if (rc) + goto ioctl_exit; rc = cifs_send_recv(xid, ses, &rqst, &resp_buftype, flags, &rsp_iov); - cifs_small_buf_release(req); rsp = (struct smb2_ioctl_rsp *)rsp_iov.iov_base; if (rc != 0) @@ -2622,6 +2642,7 @@ SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, } ioctl_exit: + SMB2_ioctl_free(&rqst); free_rsp_buf(resp_buftype, rsp); return rc; } diff --git a/fs/cifs/smb2pdu.h b/fs/cifs/smb2pdu.h index 0bd4d48027013..ee8977688e21f 100644 --- a/fs/cifs/smb2pdu.h +++ b/fs/cifs/smb2pdu.h @@ -959,6 +959,13 @@ struct duplicate_extents_to_file { __le64 ByteCount; /* Bytes to be copied */ } __packed; +/* + * Maximum number of iovs we need for an ioctl request. + * [0] : struct smb2_ioctl_req + * [1] : in_data + */ +#define SMB2_IOCTL_IOV_SIZE 2 + struct smb2_ioctl_req { struct smb2_sync_hdr sync_hdr; __le16 StructureSize; /* Must be 57 */ diff --git a/fs/cifs/smb2proto.h b/fs/cifs/smb2proto.h index 87733b27a65fe..3c32d0cfea69b 100644 --- a/fs/cifs/smb2proto.h +++ b/fs/cifs/smb2proto.h @@ -144,6 +144,10 @@ extern int SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, u64 volatile_fid, u32 opcode, bool is_fsctl, char *in_data, u32 indatalen, char **out_data, u32 *plen /* returned data len */); +extern int SMB2_ioctl_init(struct cifs_tcon *tcon, struct smb_rqst *rqst, + u64 persistent_fid, u64 volatile_fid, u32 opcode, + bool is_fsctl, char *in_data, u32 indatalen); +extern void SMB2_ioctl_free(struct smb_rqst *rqst); extern int SMB2_close(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_file_id, u64 volatile_file_id); extern int SMB2_close_flags(const unsigned int xid, struct cifs_tcon *tcon, @@ -223,6 +227,9 @@ extern int smb3_validate_negotiate(const unsigned int, struct cifs_tcon *); extern enum securityEnum smb2_select_sectype(struct TCP_Server_Info *, enum securityEnum); +extern __u8 smb2_parse_lease_state(struct TCP_Server_Info *server, + struct smb2_create_rsp *rsp, + unsigned int *epoch, char *lease_key); extern int smb3_encryption_required(const struct cifs_tcon *tcon); extern int smb2_validate_iov(unsigned int offset, unsigned int buffer_length, struct kvec *iov, unsigned int min_buf_size); diff --git a/fs/cifs/smb2status.h b/fs/cifs/smb2status.h index 3d5f62150de42..447c0c6e4c640 100644 --- a/fs/cifs/smb2status.h +++ b/fs/cifs/smb2status.h @@ -30,9 +30,9 @@ */ #define STATUS_SEVERITY_SUCCESS __constant_cpu_to_le32(0x0000) -#define STATUS_SEVERITY_INFORMATIONAL __constanst_cpu_to_le32(0x0001) -#define STATUS_SEVERITY_WARNING __constanst_cpu_to_le32(0x0002) -#define STATUS_SEVERITY_ERROR __constanst_cpu_to_le32(0x0003) +#define STATUS_SEVERITY_INFORMATIONAL cpu_to_le32(0x0001) +#define STATUS_SEVERITY_WARNING cpu_to_le32(0x0002) +#define STATUS_SEVERITY_ERROR cpu_to_le32(0x0003) struct ntstatus { /* Facility is the high 12 bits of the following field */ diff --git a/fs/cifs/trace.h b/fs/cifs/trace.h index d8b049afa6062..fa226de48ef38 100644 --- a/fs/cifs/trace.h +++ b/fs/cifs/trace.h @@ -59,6 +59,8 @@ DEFINE_EVENT(smb3_rw_err_class, smb3_##name, \ DEFINE_SMB3_RW_ERR_EVENT(write_err); DEFINE_SMB3_RW_ERR_EVENT(read_err); DEFINE_SMB3_RW_ERR_EVENT(query_dir_err); +DEFINE_SMB3_RW_ERR_EVENT(zero_err); +DEFINE_SMB3_RW_ERR_EVENT(falloc_err); /* For logging successful read or write */ @@ -104,9 +106,13 @@ DEFINE_EVENT(smb3_rw_done_class, smb3_##name, \ DEFINE_SMB3_RW_DONE_EVENT(write_enter); DEFINE_SMB3_RW_DONE_EVENT(read_enter); DEFINE_SMB3_RW_DONE_EVENT(query_dir_enter); +DEFINE_SMB3_RW_DONE_EVENT(zero_enter); +DEFINE_SMB3_RW_DONE_EVENT(falloc_enter); DEFINE_SMB3_RW_DONE_EVENT(write_done); DEFINE_SMB3_RW_DONE_EVENT(read_done); DEFINE_SMB3_RW_DONE_EVENT(query_dir_done); +DEFINE_SMB3_RW_DONE_EVENT(zero_done); +DEFINE_SMB3_RW_DONE_EVENT(falloc_done); /* * For handle based calls other than read and write, and get/set info @@ -242,6 +248,123 @@ DEFINE_SMB3_INF_ERR_EVENT(query_info_err); DEFINE_SMB3_INF_ERR_EVENT(set_info_err); DEFINE_SMB3_INF_ERR_EVENT(fsctl_err); +DECLARE_EVENT_CLASS(smb3_inf_compound_enter_class, + TP_PROTO(unsigned int xid, + __u32 tid, + __u64 sesid, + const char *full_path), + TP_ARGS(xid, tid, sesid, full_path), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u32, tid) + __field(__u64, sesid) + __string(path, full_path) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->tid = tid; + __entry->sesid = sesid; + __assign_str(path, full_path); + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x path=%s", + __entry->xid, __entry->sesid, __entry->tid, + __get_str(path)) +) + +#define DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(name) \ +DEFINE_EVENT(smb3_inf_compound_enter_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u32 tid, \ + __u64 sesid, \ + const char *full_path), \ + TP_ARGS(xid, tid, sesid, full_path)) + +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(query_info_compound_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(hardlink_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(rename_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(rmdir_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(set_eof_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(set_info_compound_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(delete_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(mkdir_enter); + + +DECLARE_EVENT_CLASS(smb3_inf_compound_done_class, + TP_PROTO(unsigned int xid, + __u32 tid, + __u64 sesid), + TP_ARGS(xid, tid, sesid), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u32, tid) + __field(__u64, sesid) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->tid = tid; + __entry->sesid = sesid; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x", + __entry->xid, __entry->sesid, __entry->tid) +) + +#define DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(name) \ +DEFINE_EVENT(smb3_inf_compound_done_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u32 tid, \ + __u64 sesid), \ + TP_ARGS(xid, tid, sesid)) + +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(query_info_compound_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(hardlink_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(rename_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(rmdir_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(set_eof_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(set_info_compound_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(delete_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(mkdir_done); + + +DECLARE_EVENT_CLASS(smb3_inf_compound_err_class, + TP_PROTO(unsigned int xid, + __u32 tid, + __u64 sesid, + int rc), + TP_ARGS(xid, tid, sesid, rc), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u32, tid) + __field(__u64, sesid) + __field(int, rc) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->rc = rc; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x rc=%d", + __entry->xid, __entry->sesid, __entry->tid, + __entry->rc) +) + +#define DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_inf_compound_err_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u32 tid, \ + __u64 sesid, \ + int rc), \ + TP_ARGS(xid, tid, sesid, rc)) + +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(query_info_compound_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(hardlink_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(rename_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(rmdir_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(set_eof_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(set_info_compound_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(mkdir_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(delete_err); + /* * For logging SMB3 Status code and Command for responses which return errors */ @@ -713,6 +836,7 @@ DEFINE_EVENT(smb3_credit_class, smb3_##name, \ TP_ARGS(currmid, hostname, credits)) DEFINE_SMB3_CREDIT_EVENT(reconnect_with_invalid_credits); +DEFINE_SMB3_CREDIT_EVENT(credit_timeout); #endif /* _CIFS_TRACE_H */ diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c index 7ce8a585abd62..1de8e996e566f 100644 --- a/fs/cifs/transport.c +++ b/fs/cifs/transport.c @@ -486,15 +486,31 @@ smb_send(struct TCP_Server_Info *server, struct smb_hdr *smb_buffer, } static int -wait_for_free_credits(struct TCP_Server_Info *server, const int timeout, - int *credits, unsigned int *instance) +wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, + const int timeout, const int flags, + unsigned int *instance) { int rc; + int *credits; + int optype; + long int t; + + if (timeout < 0) + t = MAX_JIFFY_OFFSET; + else + t = msecs_to_jiffies(timeout); + + optype = flags & CIFS_OP_MASK; *instance = 0; + credits = server->ops->get_credits_field(server, optype); + /* Since an echo is already inflight, no need to wait to send another */ + if (*credits <= 0 && optype == CIFS_ECHO_OP) + return -EAGAIN; + spin_lock(&server->req_lock); - if (timeout == CIFS_ASYNC_OP) { + if ((flags & CIFS_TIMEOUT_MASK) == CIFS_ASYNC_OP) { /* oplock breaks must not be held up */ server->in_flight++; *credits -= 1; @@ -504,14 +520,21 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int timeout, } while (1) { - if (*credits <= 0) { + if (*credits < num_credits) { spin_unlock(&server->req_lock); cifs_num_waiters_inc(server); - rc = wait_event_killable(server->request_q, - has_credits(server, credits)); + rc = wait_event_killable_timeout(server->request_q, + has_credits(server, credits, num_credits), t); cifs_num_waiters_dec(server); - if (rc) - return rc; + if (!rc) { + trace_smb3_credit_timeout(server->CurrentMid, + server->hostname, num_credits); + cifs_dbg(VFS, "wait timed out after %d ms\n", + timeout); + return -ENOTSUPP; + } + if (rc == -ERESTARTSYS) + return -ERESTARTSYS; spin_lock(&server->req_lock); } else { if (server->tcpStatus == CifsExiting) { @@ -520,14 +543,52 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int timeout, } /* + * For normal commands, reserve the last MAX_COMPOUND + * credits to compound requests. + * Otherwise these compounds could be permanently + * starved for credits by single-credit requests. + * + * To prevent spinning CPU, block this thread until + * there are >MAX_COMPOUND credits available. + * But only do this is we already have a lot of + * credits in flight to avoid triggering this check + * for servers that are slow to hand out credits on + * new sessions. + */ + if (!optype && num_credits == 1 && + server->in_flight > 2 * MAX_COMPOUND && + *credits <= MAX_COMPOUND) { + spin_unlock(&server->req_lock); + cifs_num_waiters_inc(server); + rc = wait_event_killable_timeout( + server->request_q, + has_credits(server, credits, + MAX_COMPOUND + 1), + t); + cifs_num_waiters_dec(server); + if (!rc) { + trace_smb3_credit_timeout( + server->CurrentMid, + server->hostname, num_credits); + cifs_dbg(VFS, "wait timed out after %d ms\n", + timeout); + return -ENOTSUPP; + } + if (rc == -ERESTARTSYS) + return -ERESTARTSYS; + spin_lock(&server->req_lock); + continue; + } + + /* * Can not count locking commands against total * as they are allowed to block on server. */ /* update # of requests on the wire to server */ - if (timeout != CIFS_BLOCKING_OP) { - *credits -= 1; - server->in_flight++; + if ((flags & CIFS_TIMEOUT_MASK) != CIFS_BLOCKING_OP) { + *credits -= num_credits; + server->in_flight += num_credits; *instance = server->reconnect_instance; } spin_unlock(&server->req_lock); @@ -538,16 +599,36 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int timeout, } static int -wait_for_free_request(struct TCP_Server_Info *server, const int timeout, - const int optype, unsigned int *instance) +wait_for_free_request(struct TCP_Server_Info *server, const int flags, + unsigned int *instance) { - int *val; + return wait_for_free_credits(server, 1, -1, flags, + instance); +} - val = server->ops->get_credits_field(server, optype); - /* Since an echo is already inflight, no need to wait to send another */ - if (*val <= 0 && optype == CIFS_ECHO_OP) - return -EAGAIN; - return wait_for_free_credits(server, timeout, val, instance); +static int +wait_for_compound_request(struct TCP_Server_Info *server, int num, + const int flags, unsigned int *instance) +{ + int *credits; + + credits = server->ops->get_credits_field(server, flags & CIFS_OP_MASK); + + spin_lock(&server->req_lock); + if (*credits < num) { + /* + * Return immediately if not too many requests in flight since + * we will likely be stuck on waiting for credits. + */ + if (server->in_flight < num - *credits) { + spin_unlock(&server->req_lock); + return -ENOTSUPP; + } + } + spin_unlock(&server->req_lock); + + return wait_for_free_credits(server, num, 60000, flags, + instance); } int @@ -646,16 +727,16 @@ cifs_call_async(struct TCP_Server_Info *server, struct smb_rqst *rqst, mid_handle_t *handle, void *cbdata, const int flags, const struct cifs_credits *exist_credits) { - int rc, timeout, optype; + int rc; struct mid_q_entry *mid; struct cifs_credits credits = { .value = 0, .instance = 0 }; unsigned int instance; + int optype; - timeout = flags & CIFS_TIMEOUT_MASK; optype = flags & CIFS_OP_MASK; if ((flags & CIFS_HAS_CREDITS) == 0) { - rc = wait_for_free_request(server, timeout, optype, &instance); + rc = wait_for_free_request(server, flags, &instance); if (rc) return rc; credits.value = 1; @@ -871,18 +952,15 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, const int flags, const int num_rqst, struct smb_rqst *rqst, int *resp_buf_type, struct kvec *resp_iov) { - int i, j, rc = 0; - int timeout, optype; + int i, j, optype, rc = 0; struct mid_q_entry *midQ[MAX_COMPOUND]; bool cancelled_mid[MAX_COMPOUND] = {false}; struct cifs_credits credits[MAX_COMPOUND] = { { .value = 0, .instance = 0 } }; unsigned int instance; - unsigned int first_instance = 0; char *buf; - timeout = flags & CIFS_TIMEOUT_MASK; optype = flags & CIFS_OP_MASK; for (i = 0; i < num_rqst; i++) @@ -896,81 +974,24 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, if (ses->server->tcpStatus == CifsExiting) return -ENOENT; - spin_lock(&ses->server->req_lock); - if (ses->server->credits < num_rqst) { - /* - * Return immediately if not too many requests in flight since - * we will likely be stuck on waiting for credits. - */ - if (ses->server->in_flight < num_rqst - ses->server->credits) { - spin_unlock(&ses->server->req_lock); - return -ENOTSUPP; - } - } else { - /* enough credits to send the whole compounded request */ - ses->server->credits -= num_rqst; - ses->server->in_flight += num_rqst; - first_instance = ses->server->reconnect_instance; - } - spin_unlock(&ses->server->req_lock); - - if (first_instance) { - cifs_dbg(FYI, "Acquired %d credits at once\n", num_rqst); - for (i = 0; i < num_rqst; i++) { - credits[i].value = 1; - credits[i].instance = first_instance; - } - goto setup_rqsts; - } - /* - * There are not enough credits to send the whole compound request but - * there are requests in flight that may bring credits from the server. + * Wait for all the requests to become available. * This approach still leaves the possibility to be stuck waiting for * credits if the server doesn't grant credits to the outstanding - * requests. This should be fixed by returning immediately and letting - * a caller fallback to sequential commands instead of compounding. - * Ensure we obtain 1 credit per request in the compound chain. + * requests and if the client is completely idle, not generating any + * other requests. + * This can be handled by the eventual session reconnect. */ - for (i = 0; i < num_rqst; i++) { - rc = wait_for_free_request(ses->server, timeout, optype, - &instance); - - if (rc == 0) { - credits[i].value = 1; - credits[i].instance = instance; - /* - * All parts of the compound chain must get credits from - * the same session, otherwise we may end up using more - * credits than the server granted. If there were - * reconnects in between, return -EAGAIN and let callers - * handle it. - */ - if (i == 0) - first_instance = instance; - else if (first_instance != instance) { - i++; - rc = -EAGAIN; - } - } + rc = wait_for_compound_request(ses->server, num_rqst, flags, + &instance); + if (rc) + return rc; - if (rc) { - /* - * We haven't sent an SMB packet to the server yet but - * we already obtained credits for i requests in the - * compound chain - need to return those credits back - * for future use. Note that we need to call add_credits - * multiple times to match the way we obtained credits - * in the first place and to account for in flight - * requests correctly. - */ - for (j = 0; j < i; j++) - add_credits(ses->server, &credits[j], optype); - return rc; - } + for (i = 0; i < num_rqst; i++) { + credits[i].value = 1; + credits[i].instance = instance; } -setup_rqsts: /* * Make sure that we sign in the same order that we send on this socket * and avoid races inside tcp sendmsg code that could cause corruption @@ -981,14 +1002,12 @@ setup_rqsts: /* * All the parts of the compound chain belong obtained credits from the - * same session (see the appropriate checks above). In the same time - * there might be reconnects after those checks but before we acquired - * the srv_mutex. We can not use credits obtained from the previous + * same session. We can not use credits obtained from the previous * session to send this request. Check if there were reconnects after * we obtained credits and return -EAGAIN in such cases to let callers * handle it. */ - if (first_instance != ses->server->reconnect_instance) { + if (instance != ses->server->reconnect_instance) { mutex_unlock(&ses->server->srv_mutex); for (j = 0; j < num_rqst; j++) add_credits(ses->server, &credits[j], optype); @@ -1057,7 +1076,7 @@ setup_rqsts: smb311_update_preauth_hash(ses, rqst[0].rq_iov, rqst[0].rq_nvec); - if (timeout == CIFS_ASYNC_OP) + if ((flags & CIFS_TIMEOUT_MASK) == CIFS_ASYNC_OP) goto out; for (i = 0; i < num_rqst; i++) { @@ -1194,7 +1213,7 @@ SendReceive2(const unsigned int xid, struct cifs_ses *ses, int SendReceive(const unsigned int xid, struct cifs_ses *ses, struct smb_hdr *in_buf, struct smb_hdr *out_buf, - int *pbytes_returned, const int timeout) + int *pbytes_returned, const int flags) { int rc = 0; struct mid_q_entry *midQ; @@ -1225,7 +1244,7 @@ SendReceive(const unsigned int xid, struct cifs_ses *ses, return -EIO; } - rc = wait_for_free_request(ses->server, timeout, 0, &credits.instance); + rc = wait_for_free_request(ses->server, flags, &credits.instance); if (rc) return rc; @@ -1264,7 +1283,7 @@ SendReceive(const unsigned int xid, struct cifs_ses *ses, if (rc < 0) goto out; - if (timeout == CIFS_ASYNC_OP) + if ((flags & CIFS_TIMEOUT_MASK) == CIFS_ASYNC_OP) goto out; rc = wait_for_response(ses->server, midQ); @@ -1367,8 +1386,7 @@ SendReceiveBlockingLock(const unsigned int xid, struct cifs_tcon *tcon, return -EIO; } - rc = wait_for_free_request(ses->server, CIFS_BLOCKING_OP, 0, - &instance); + rc = wait_for_free_request(ses->server, CIFS_BLOCKING_OP, &instance); if (rc) return rc; diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index f955cd3e06779..a98e1b02279ea 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -306,8 +306,9 @@ static int f2fs_write_meta_pages(struct address_space *mapping, goto skip_write; /* collect a number of dirty meta pages and write together */ - if (wbc->for_kupdate || - get_pages(sbi, F2FS_DIRTY_META) < nr_pages_to_skip(sbi, META)) + if (wbc->sync_mode != WB_SYNC_ALL && + get_pages(sbi, F2FS_DIRTY_META) < + nr_pages_to_skip(sbi, META)) goto skip_write; /* if locked failed, cp will flush dirty pages instead */ @@ -405,7 +406,7 @@ static int f2fs_set_meta_page_dirty(struct page *page) if (!PageDirty(page)) { __set_page_dirty_nobuffers(page); inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_META); - SetPagePrivate(page); + f2fs_set_page_private(page, 0); f2fs_trace_pid(page); return 1; } @@ -956,7 +957,7 @@ void f2fs_update_dirty_page(struct inode *inode, struct page *page) inode_inc_dirty_pages(inode); spin_unlock(&sbi->inode_lock[type]); - SetPagePrivate(page); + f2fs_set_page_private(page, 0); f2fs_trace_pid(page); } @@ -1259,10 +1260,17 @@ static void update_ckpt_flags(struct f2fs_sb_info *sbi, struct cp_control *cpc) else __clear_ckpt_flags(ckpt, CP_DISABLED_FLAG); + if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK)) + __set_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG); + else + __clear_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG); + if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH)) __set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG); - else - __clear_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG); + /* + * TODO: we count on fsck.f2fs to clear this flag until we figure out + * missing cases which clear it incorrectly. + */ if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR)) __set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 568e1d09eb486..9727944139f21 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -301,9 +301,10 @@ static inline void __submit_bio(struct f2fs_sb_info *sbi, for (; start < F2FS_IO_SIZE(sbi); start++) { struct page *page = mempool_alloc(sbi->write_io_dummy, - GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL); + GFP_NOIO | __GFP_NOFAIL); f2fs_bug_on(sbi, !page); + zero_user_segment(page, 0, PAGE_SIZE); SetPagePrivate(page); set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE); lock_page(page); @@ -1553,6 +1554,9 @@ static int f2fs_mpage_readpages(struct address_space *mapping, if (last_block > last_block_in_file) last_block = last_block_in_file; + /* just zeroing out page which is beyond EOF */ + if (block_in_file >= last_block) + goto zero_out; /* * Map blocks using the previous result first. */ @@ -1565,16 +1569,11 @@ static int f2fs_mpage_readpages(struct address_space *mapping, * Then do more f2fs_map_blocks() calls until we are * done with this page. */ - map.m_flags = 0; - - if (block_in_file < last_block) { - map.m_lblk = block_in_file; - map.m_len = last_block - block_in_file; + map.m_lblk = block_in_file; + map.m_len = last_block - block_in_file; - if (f2fs_map_blocks(inode, &map, 0, - F2FS_GET_BLOCK_DEFAULT)) - goto set_error_page; - } + if (f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT)) + goto set_error_page; got_it: if ((map.m_flags & F2FS_MAP_MAPPED)) { block_nr = map.m_pblk + block_in_file - map.m_lblk; @@ -1589,6 +1588,7 @@ got_it: DATA_GENERIC)) goto set_error_page; } else { +zero_out: zero_user_segment(page, 0, PAGE_SIZE); if (!PageUptodate(page)) SetPageUptodate(page); @@ -1863,8 +1863,13 @@ got_it: if (fio->need_lock == LOCK_REQ) f2fs_unlock_op(fio->sbi); err = f2fs_inplace_write_data(fio); - if (err && PageWriteback(page)) - end_page_writeback(page); + if (err) { + if (f2fs_encrypted_file(inode)) + fscrypt_pullback_bio_page(&fio->encrypted_page, + true); + if (PageWriteback(page)) + end_page_writeback(page); + } trace_f2fs_do_write_data_page(fio->page, IPU); set_inode_flag(inode, FI_UPDATE_WRITE); return err; @@ -2315,7 +2320,8 @@ static void f2fs_write_failed(struct address_space *mapping, loff_t to) down_write(&F2FS_I(inode)->i_mmap_sem); truncate_pagecache(inode, i_size); - f2fs_truncate_blocks(inode, i_size, true, true); + if (!IS_NOQUOTA(inode)) + f2fs_truncate_blocks(inode, i_size, true); up_write(&F2FS_I(inode)->i_mmap_sem); up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); @@ -2585,14 +2591,11 @@ static void f2fs_dio_submit_bio(struct bio *bio, struct inode *inode, { struct f2fs_private_dio *dio; bool write = (bio_op(bio) == REQ_OP_WRITE); - int err; dio = f2fs_kzalloc(F2FS_I_SB(inode), sizeof(struct f2fs_private_dio), GFP_NOFS); - if (!dio) { - err = -ENOMEM; + if (!dio) goto out; - } dio->inode = inode; dio->orig_end_io = bio->bi_end_io; @@ -2710,12 +2713,10 @@ void f2fs_invalidate_page(struct page *page, unsigned int offset, clear_cold_data(page); - /* This is atomic written page, keep Private */ if (IS_ATOMIC_WRITTEN_PAGE(page)) return f2fs_drop_inmem_page(inode, page); - set_page_private(page, 0); - ClearPagePrivate(page); + f2fs_clear_page_private(page); } int f2fs_release_page(struct page *page, gfp_t wait) @@ -2729,8 +2730,7 @@ int f2fs_release_page(struct page *page, gfp_t wait) return 0; clear_cold_data(page); - set_page_private(page, 0); - ClearPagePrivate(page); + f2fs_clear_page_private(page); return 1; } @@ -2798,12 +2798,8 @@ int f2fs_migrate_page(struct address_space *mapping, return -EAGAIN; } - /* - * A reference is expected if PagePrivate set when move mapping, - * however F2FS breaks this for maintaining dirty page counts when - * truncating pages. So here adjusting the 'extra_count' make it work. - */ - extra_count = (atomic_written ? 1 : 0) - page_has_private(page); + /* one extra reference was held for atomic_write page */ + extra_count = atomic_written ? 1 : 0; rc = migrate_page_move_mapping(mapping, newpage, page, mode, extra_count); if (rc != MIGRATEPAGE_SUCCESS) { @@ -2824,9 +2820,10 @@ int f2fs_migrate_page(struct address_space *mapping, get_page(newpage); } - if (PagePrivate(page)) - SetPagePrivate(newpage); - set_page_private(newpage, page_private(page)); + if (PagePrivate(page)) { + f2fs_set_page_private(newpage, page_private(page)); + f2fs_clear_page_private(page); + } if (mode != MIGRATE_SYNC_NO_COPY) migrate_page_copy(newpage, page); diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index fd7f170e2f2d6..99e9a5c37b711 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -96,8 +96,10 @@ static void update_general_status(struct f2fs_sb_info *sbi) si->free_secs = free_sections(sbi); si->prefree_count = prefree_segments(sbi); si->dirty_count = dirty_segments(sbi); - si->node_pages = NODE_MAPPING(sbi)->nrpages; - si->meta_pages = META_MAPPING(sbi)->nrpages; + if (sbi->node_inode) + si->node_pages = NODE_MAPPING(sbi)->nrpages; + if (sbi->meta_inode) + si->meta_pages = META_MAPPING(sbi)->nrpages; si->nats = NM_I(sbi)->nat_cnt; si->dirty_nats = NM_I(sbi)->dirty_nat_cnt; si->sits = MAIN_SEGS(sbi); @@ -175,7 +177,6 @@ static void update_sit_info(struct f2fs_sb_info *sbi) static void update_mem_info(struct f2fs_sb_info *sbi) { struct f2fs_stat_info *si = F2FS_STAT(sbi); - unsigned npages; int i; if (si->base_mem) @@ -258,10 +259,14 @@ get_cache: sizeof(struct extent_node); si->page_mem = 0; - npages = NODE_MAPPING(sbi)->nrpages; - si->page_mem += (unsigned long long)npages << PAGE_SHIFT; - npages = META_MAPPING(sbi)->nrpages; - si->page_mem += (unsigned long long)npages << PAGE_SHIFT; + if (sbi->node_inode) { + unsigned npages = NODE_MAPPING(sbi)->nrpages; + si->page_mem += (unsigned long long)npages << PAGE_SHIFT; + } + if (sbi->meta_inode) { + unsigned npages = META_MAPPING(sbi)->nrpages; + si->page_mem += (unsigned long long)npages << PAGE_SHIFT; + } } static int stat_show(struct seq_file *s, void *v) diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index 713b36a10a790..59bc460178554 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -728,7 +728,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page, !f2fs_truncate_hole(dir, page->index, page->index + 1)) { f2fs_clear_page_cache_dirty_tag(page); clear_page_dirty_for_io(page); - ClearPagePrivate(page); + f2fs_clear_page_private(page); ClearPageUptodate(page); clear_cold_data(page); inode_dec_dirty_pages(dir); @@ -800,6 +800,10 @@ int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, if (de->name_len == 0) { bit_pos++; ctx->pos = start_pos + bit_pos; + printk_ratelimited( + "%s, invalid namelen(0), ino:%u, run fsck to fix.", + KERN_WARNING, le32_to_cpu(de->ino)); + set_sbi_flag(sbi, SBI_NEED_FSCK); continue; } @@ -810,7 +814,8 @@ int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, /* check memory boundary before moving forward */ bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)); - if (unlikely(bit_pos > d->max)) { + if (unlikely(bit_pos > d->max || + le16_to_cpu(de->name_len) > F2FS_NAME_LEN)) { f2fs_msg(sbi->sb, KERN_WARNING, "%s: corrupted namelen=%d, run fsck to fix.", __func__, le16_to_cpu(de->name_len)); @@ -891,7 +896,7 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx) page_cache_sync_readahead(inode->i_mapping, ra, file, n, min(npages - n, (pgoff_t)MAX_DIR_RA_PAGES)); - dentry_page = f2fs_get_lock_data_page(inode, n, false); + dentry_page = f2fs_find_data_page(inode, n); if (IS_ERR(dentry_page)) { err = PTR_ERR(dentry_page); if (err == -ENOENT) { @@ -909,11 +914,11 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx) err = f2fs_fill_dentries(ctx, &d, n * NR_DENTRY_IN_BLOCK, &fstr); if (err) { - f2fs_put_page(dentry_page, 1); + f2fs_put_page(dentry_page, 0); break; } - f2fs_put_page(dentry_page, 1); + f2fs_put_page(dentry_page, 0); } out_free: fscrypt_fname_free_buffer(&fstr); diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index 1cb0fcc67d2df..caf77fe8ac073 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -506,7 +506,7 @@ static void f2fs_update_extent_tree_range(struct inode *inode, unsigned int end = fofs + len; unsigned int pos = (unsigned int)fofs; bool updated = false; - bool leftmost; + bool leftmost = false; if (!et) return; diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 7ea5c9cede371..87f75ebd2fd60 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -190,6 +190,8 @@ enum { #define DEF_CP_INTERVAL 60 /* 60 secs */ #define DEF_IDLE_INTERVAL 5 /* 5 secs */ #define DEF_DISABLE_INTERVAL 5 /* 5 secs */ +#define DEF_DISABLE_QUICK_INTERVAL 1 /* 1 secs */ +#define DEF_UMOUNT_DISCARD_TIMEOUT 5 /* 5 secs */ struct cp_control { int reason; @@ -253,7 +255,7 @@ struct discard_entry { /* max discard pend list number */ #define MAX_PLIST_NUM 512 #define plist_idx(blk_num) ((blk_num) >= MAX_PLIST_NUM ? \ - (MAX_PLIST_NUM - 1) : (blk_num - 1)) + (MAX_PLIST_NUM - 1) : ((blk_num) - 1)) enum { D_PREP, /* initial */ @@ -309,6 +311,7 @@ struct discard_policy { bool sync; /* submit discard with REQ_SYNC flag */ bool ordered; /* issue discard by lba order */ unsigned int granularity; /* discard granularity */ + int timeout; /* discard timeout for put_super */ }; struct discard_cmd_control { @@ -455,7 +458,6 @@ struct f2fs_flush_device { /* for inline stuff */ #define DEF_INLINE_RESERVED_SIZE 1 -#define DEF_MIN_INLINE_SIZE 1 static inline int get_extra_isize(struct inode *inode); static inline int get_inline_xattr_addrs(struct inode *inode); #define MAX_INLINE_DATA(inode) (sizeof(__le32) * \ @@ -1098,6 +1100,7 @@ enum { SBI_IS_SHUTDOWN, /* shutdown by ioctl */ SBI_IS_RECOVERED, /* recovered orphan/data */ SBI_CP_DISABLED, /* CP was disabled last mount */ + SBI_CP_DISABLED_QUICK, /* CP was disabled quickly */ SBI_QUOTA_NEED_FLUSH, /* need to flush quota info in CP */ SBI_QUOTA_SKIP_FLUSH, /* skip flushing quota in current CP */ SBI_QUOTA_NEED_REPAIR, /* quota file may be corrupted */ @@ -1109,6 +1112,7 @@ enum { DISCARD_TIME, GC_TIME, DISABLE_TIME, + UMOUNT_DISCARD_TIMEOUT, MAX_TIME, }; @@ -1237,8 +1241,6 @@ struct f2fs_sb_info { unsigned int nquota_files; /* # of quota sysfile */ - u32 s_next_generation; /* for NFS support */ - /* # of pages, see count_type */ atomic_t nr_pages[NR_COUNT_TYPE]; /* # of allocated blocks */ @@ -1798,13 +1800,12 @@ static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type) { atomic_inc(&sbi->nr_pages[count_type]); - if (count_type == F2FS_DIRTY_DATA || count_type == F2FS_INMEM_PAGES || - count_type == F2FS_WB_CP_DATA || count_type == F2FS_WB_DATA || - count_type == F2FS_RD_DATA || count_type == F2FS_RD_NODE || - count_type == F2FS_RD_META) - return; - - set_sbi_flag(sbi, SBI_IS_DIRTY); + if (count_type == F2FS_DIRTY_DENTS || + count_type == F2FS_DIRTY_NODES || + count_type == F2FS_DIRTY_META || + count_type == F2FS_DIRTY_QDATA || + count_type == F2FS_DIRTY_IMETA) + set_sbi_flag(sbi, SBI_IS_DIRTY); } static inline void inode_inc_dirty_pages(struct inode *inode) @@ -2156,10 +2157,17 @@ static inline bool is_idle(struct f2fs_sb_info *sbi, int type) get_pages(sbi, F2FS_RD_META) || get_pages(sbi, F2FS_WB_DATA) || get_pages(sbi, F2FS_WB_CP_DATA) || get_pages(sbi, F2FS_DIO_READ) || - get_pages(sbi, F2FS_DIO_WRITE) || - atomic_read(&SM_I(sbi)->dcc_info->queued_discard) || - atomic_read(&SM_I(sbi)->fcc_info->queued_flush)) + get_pages(sbi, F2FS_DIO_WRITE)) return false; + + if (SM_I(sbi) && SM_I(sbi)->dcc_info && + atomic_read(&SM_I(sbi)->dcc_info->queued_discard)) + return false; + + if (SM_I(sbi) && SM_I(sbi)->fcc_info && + atomic_read(&SM_I(sbi)->fcc_info->queued_flush)) + return false; + return f2fs_time_over(sbi, type); } @@ -2300,11 +2308,12 @@ static inline void f2fs_change_bit(unsigned int nr, char *addr) #define F2FS_EXTENTS_FL 0x00080000 /* Inode uses extents */ #define F2FS_EA_INODE_FL 0x00200000 /* Inode used for large EA */ #define F2FS_EOFBLOCKS_FL 0x00400000 /* Blocks allocated beyond EOF */ +#define F2FS_NOCOW_FL 0x00800000 /* Do not cow file */ #define F2FS_INLINE_DATA_FL 0x10000000 /* Inode has inline data. */ #define F2FS_PROJINHERIT_FL 0x20000000 /* Create with parents projid */ #define F2FS_RESERVED_FL 0x80000000 /* reserved for ext4 lib */ -#define F2FS_FL_USER_VISIBLE 0x304BDFFF /* User visible flags */ +#define F2FS_FL_USER_VISIBLE 0x30CBDFFF /* User visible flags */ #define F2FS_FL_USER_MODIFIABLE 0x204BC0FF /* User modifiable flags */ /* Flags we can manipulate with through F2FS_IOC_FSSETXATTR */ @@ -2761,9 +2770,9 @@ static inline int get_inline_xattr_addrs(struct inode *inode) #define F2FS_OLD_ATTRIBUTE_SIZE (offsetof(struct f2fs_inode, i_addr)) #define F2FS_FITS_IN_INODE(f2fs_inode, extra_isize, field) \ - ((offsetof(typeof(*f2fs_inode), field) + \ + ((offsetof(typeof(*(f2fs_inode)), field) + \ sizeof((f2fs_inode)->field)) \ - <= (F2FS_OLD_ATTRIBUTE_SIZE + extra_isize)) \ + <= (F2FS_OLD_ATTRIBUTE_SIZE + (extra_isize))) \ static inline void f2fs_reset_iostat(struct f2fs_sb_info *sbi) { @@ -2792,8 +2801,8 @@ static inline void f2fs_update_iostat(struct f2fs_sb_info *sbi, #define __is_large_section(sbi) ((sbi)->segs_per_sec > 1) -#define __is_meta_io(fio) (PAGE_TYPE_OF_BIO(fio->type) == META && \ - (!is_read_io(fio->op) || fio->is_meta)) +#define __is_meta_io(fio) (PAGE_TYPE_OF_BIO((fio)->type) == META && \ + (!is_read_io((fio)->op) || (fio)->is_meta)) bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr, int type); @@ -2825,13 +2834,33 @@ static inline bool is_valid_data_blkaddr(struct f2fs_sb_info *sbi, return true; } +static inline void f2fs_set_page_private(struct page *page, + unsigned long data) +{ + if (PagePrivate(page)) + return; + + get_page(page); + SetPagePrivate(page); + set_page_private(page, data); +} + +static inline void f2fs_clear_page_private(struct page *page) +{ + if (!PagePrivate(page)) + return; + + set_page_private(page, 0); + ClearPagePrivate(page); + f2fs_put_page(page, 0); +} + /* * file.c */ int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync); void f2fs_truncate_data_blocks(struct dnode_of_data *dn); -int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock, - bool buf_write); +int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock); int f2fs_truncate(struct inode *inode); int f2fs_getattr(const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags); @@ -3005,7 +3034,7 @@ void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr); bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr); void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi); void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi); -bool f2fs_wait_discard_bios(struct f2fs_sb_info *sbi); +bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi); void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc); void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi); @@ -3610,8 +3639,6 @@ extern void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate, #define f2fs_build_fault_attr(sbi, rate, type) do { } while (0) #endif -#endif - static inline bool is_journalled_quota(struct f2fs_sb_info *sbi) { #ifdef CONFIG_QUOTA @@ -3624,3 +3651,5 @@ static inline bool is_journalled_quota(struct f2fs_sb_info *sbi) #endif return false; } + +#endif diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index ba5954f41e140..5742ab8b57dc6 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -589,8 +589,7 @@ truncate_out: return 0; } -int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock, - bool buf_write) +int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct dnode_of_data dn; @@ -598,7 +597,6 @@ int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock, int count = 0, err = 0; struct page *ipage; bool truncate_page = false; - int flag = buf_write ? F2FS_GET_BLOCK_PRE_AIO : F2FS_GET_BLOCK_PRE_DIO; trace_f2fs_truncate_blocks_enter(inode, from); @@ -608,7 +606,7 @@ int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock, goto free_partial; if (lock) - __do_map_lock(sbi, flag, true); + f2fs_lock_op(sbi); ipage = f2fs_get_node_page(sbi, inode->i_ino); if (IS_ERR(ipage)) { @@ -646,7 +644,7 @@ free_next: err = f2fs_truncate_inode_blocks(inode, free_from); out: if (lock) - __do_map_lock(sbi, flag, false); + f2fs_unlock_op(sbi); free_partial: /* lastly zero out the first data page */ if (!err) @@ -681,7 +679,7 @@ int f2fs_truncate(struct inode *inode) return err; } - err = f2fs_truncate_blocks(inode, i_size_read(inode), true, false); + err = f2fs_truncate_blocks(inode, i_size_read(inode), true); if (err) return err; @@ -768,7 +766,6 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) { struct inode *inode = d_inode(dentry); int err; - bool size_changed = false; if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) return -EIO; @@ -843,8 +840,6 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) down_write(&F2FS_I(inode)->i_sem); F2FS_I(inode)->last_disk_size = i_size_read(inode); up_write(&F2FS_I(inode)->i_sem); - - size_changed = true; } __setattr_copy(inode, attr); @@ -858,7 +853,7 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) } /* file size may changed here */ - f2fs_mark_inode_dirty_sync(inode, size_changed); + f2fs_mark_inode_dirty_sync(inode, true); /* inode change will produce dirty node pages flushed by checkpoint */ f2fs_balance_fs(F2FS_I_SB(inode), true); @@ -1262,7 +1257,7 @@ static int f2fs_collapse_range(struct inode *inode, loff_t offset, loff_t len) new_size = i_size_read(inode) - len; truncate_pagecache(inode, new_size); - ret = f2fs_truncate_blocks(inode, new_size, true, false); + ret = f2fs_truncate_blocks(inode, new_size, true); up_write(&F2FS_I(inode)->i_mmap_sem); if (!ret) f2fs_i_size_write(inode, new_size); @@ -1447,7 +1442,7 @@ static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) f2fs_balance_fs(sbi, true); down_write(&F2FS_I(inode)->i_mmap_sem); - ret = f2fs_truncate_blocks(inode, i_size_read(inode), true, false); + ret = f2fs_truncate_blocks(inode, i_size_read(inode), true); up_write(&F2FS_I(inode)->i_mmap_sem); if (ret) return ret; @@ -1651,6 +1646,8 @@ static int f2fs_ioc_getflags(struct file *filp, unsigned long arg) flags |= F2FS_ENCRYPT_FL; if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) flags |= F2FS_INLINE_DATA_FL; + if (is_inode_flag_set(inode, FI_PIN_FILE)) + flags |= F2FS_NOCOW_FL; flags &= F2FS_FL_USER_VISIBLE; @@ -1750,10 +1747,12 @@ static int f2fs_ioc_start_atomic_write(struct file *filp) down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); - if (!get_dirty_pages(inode)) - goto skip_flush; - - f2fs_msg(F2FS_I_SB(inode)->sb, KERN_WARNING, + /* + * Should wait end_io to count F2FS_WB_CP_DATA correctly by + * f2fs_is_atomic_file. + */ + if (get_dirty_pages(inode)) + f2fs_msg(F2FS_I_SB(inode)->sb, KERN_WARNING, "Unexpected flush for atomic writes: ino=%lu, npages=%u", inode->i_ino, get_dirty_pages(inode)); ret = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); @@ -1761,7 +1760,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp) up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); goto out; } -skip_flush: + set_inode_flag(inode, FI_ATOMIC_FILE); clear_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST); up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); @@ -1968,11 +1967,11 @@ static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) break; case F2FS_GOING_DOWN_NEED_FSCK: set_sbi_flag(sbi, SBI_NEED_FSCK); + set_sbi_flag(sbi, SBI_CP_DISABLED_QUICK); + set_sbi_flag(sbi, SBI_IS_DIRTY); /* do checkpoint only */ ret = f2fs_sync_fs(sb, 1); - if (ret) - goto out; - break; + goto out; default: ret = -EINVAL; goto out; @@ -1988,6 +1987,9 @@ static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) out: if (in != F2FS_GOING_DOWN_FULLSYNC) mnt_drop_write_file(filp); + + trace_f2fs_shutdown(sbi, in, ret); + return ret; } @@ -2871,8 +2873,8 @@ static int f2fs_ioc_set_pin_file(struct file *filp, unsigned long arg) __u32 pin; int ret = 0; - if (!inode_owner_or_capable(inode)) - return -EACCES; + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; if (get_user(pin, (__u32 __user *)arg)) return -EFAULT; diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index d636cbcf68f2e..bb6a152310ef4 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -298,7 +298,7 @@ process_inline: clear_inode_flag(inode, FI_INLINE_DATA); f2fs_put_page(ipage, 1); } else if (ri && (ri->i_inline & F2FS_INLINE_DATA)) { - if (f2fs_truncate_blocks(inode, 0, false, false)) + if (f2fs_truncate_blocks(inode, 0, false)) return false; goto process_inline; } @@ -470,7 +470,7 @@ static int f2fs_add_inline_entries(struct inode *dir, void *inline_dentry) return 0; punch_dentry_pages: truncate_inode_pages(&dir->i_data, 0); - f2fs_truncate_blocks(dir, 0, false, false); + f2fs_truncate_blocks(dir, 0, false); f2fs_remove_dirty_inode(dir); return err; } @@ -659,6 +659,12 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx, if (IS_ERR(ipage)) return PTR_ERR(ipage); + /* + * f2fs_readdir was protected by inode.i_rwsem, it is safe to access + * ipage without page's lock held. + */ + unlock_page(ipage); + inline_dentry = inline_data_addr(inode, ipage); make_dentry_ptr_inline(inode, &d, inline_dentry); @@ -667,7 +673,7 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx, if (!err) ctx->pos = d.max; - f2fs_put_page(ipage, 1); + f2fs_put_page(ipage, 0); return err < 0 ? err : 0; } diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index d910a820ae675..e7f2e87593156 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -14,6 +14,7 @@ #include "f2fs.h" #include "node.h" #include "segment.h" +#include "xattr.h" #include <trace/events/f2fs.h> @@ -248,6 +249,20 @@ static bool sanity_check_inode(struct inode *inode, struct page *node_page) return false; } + if (f2fs_has_extra_attr(inode) && + f2fs_sb_has_flexible_inline_xattr(sbi) && + f2fs_has_inline_xattr(inode) && + (!fi->i_inline_xattr_size || + fi->i_inline_xattr_size > MAX_INLINE_XATTR_SIZE)) { + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_msg(sbi->sb, KERN_WARNING, + "%s: inode (ino=%lx) has corrupted " + "i_inline_xattr_size: %d, max: %zu", + __func__, inode->i_ino, fi->i_inline_xattr_size, + MAX_INLINE_XATTR_SIZE); + return false; + } + if (F2FS_I(inode)->extent_tree) { struct extent_info *ei = &F2FS_I(inode)->extent_tree->largest; diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index e967d27c1a890..f5e34e4670031 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -10,6 +10,7 @@ #include <linux/pagemap.h> #include <linux/sched.h> #include <linux/ctype.h> +#include <linux/random.h> #include <linux/dcache.h> #include <linux/namei.h> #include <linux/quotaops.h> @@ -50,7 +51,7 @@ static struct inode *f2fs_new_inode(struct inode *dir, umode_t mode) inode->i_blocks = 0; inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); F2FS_I(inode)->i_crtime = inode->i_mtime; - inode->i_generation = sbi->s_next_generation++; + inode->i_generation = prandom_u32(); if (S_ISDIR(inode->i_mode)) F2FS_I(inode)->i_current_depth = 1; diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 4f450e5733124..3f99ab2886955 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -1920,7 +1920,9 @@ static int f2fs_write_node_pages(struct address_space *mapping, f2fs_balance_fs_bg(sbi); /* collect a number of dirty node pages and write together */ - if (get_pages(sbi, F2FS_DIRTY_NODES) < nr_pages_to_skip(sbi, NODE)) + if (wbc->sync_mode != WB_SYNC_ALL && + get_pages(sbi, F2FS_DIRTY_NODES) < + nr_pages_to_skip(sbi, NODE)) goto skip_write; if (wbc->sync_mode == WB_SYNC_ALL) @@ -1959,7 +1961,7 @@ static int f2fs_set_node_page_dirty(struct page *page) if (!PageDirty(page)) { __set_page_dirty_nobuffers(page); inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_NODES); - SetPagePrivate(page); + f2fs_set_page_private(page, 0); f2fs_trace_pid(page); return 1; } diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 9b79056d705d2..aa7fe79b62b2e 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -191,8 +191,7 @@ void f2fs_register_inmem_page(struct inode *inode, struct page *page) f2fs_trace_pid(page); - set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE); - SetPagePrivate(page); + f2fs_set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE); new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS); @@ -215,7 +214,8 @@ void f2fs_register_inmem_page(struct inode *inode, struct page *page) } static int __revoke_inmem_pages(struct inode *inode, - struct list_head *head, bool drop, bool recover) + struct list_head *head, bool drop, bool recover, + bool trylock) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct inmem_pages *cur, *tmp; @@ -227,7 +227,16 @@ static int __revoke_inmem_pages(struct inode *inode, if (drop) trace_f2fs_commit_inmem_page(page, INMEM_DROP); - lock_page(page); + if (trylock) { + /* + * to avoid deadlock in between page lock and + * inmem_lock. + */ + if (!trylock_page(page)) + continue; + } else { + lock_page(page); + } f2fs_wait_on_page_writeback(page, DATA, true, true); @@ -270,8 +279,7 @@ next: ClearPageUptodate(page); clear_cold_data(page); } - set_page_private(page, 0); - ClearPagePrivate(page); + f2fs_clear_page_private(page); f2fs_put_page(page, 1); list_del(&cur->list); @@ -318,13 +326,19 @@ void f2fs_drop_inmem_pages(struct inode *inode) struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); - mutex_lock(&fi->inmem_lock); - __revoke_inmem_pages(inode, &fi->inmem_pages, true, false); - spin_lock(&sbi->inode_lock[ATOMIC_FILE]); - if (!list_empty(&fi->inmem_ilist)) - list_del_init(&fi->inmem_ilist); - spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); - mutex_unlock(&fi->inmem_lock); + while (!list_empty(&fi->inmem_pages)) { + mutex_lock(&fi->inmem_lock); + __revoke_inmem_pages(inode, &fi->inmem_pages, + true, false, true); + + if (list_empty(&fi->inmem_pages)) { + spin_lock(&sbi->inode_lock[ATOMIC_FILE]); + if (!list_empty(&fi->inmem_ilist)) + list_del_init(&fi->inmem_ilist); + spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); + } + mutex_unlock(&fi->inmem_lock); + } clear_inode_flag(inode, FI_ATOMIC_FILE); fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0; @@ -354,8 +368,7 @@ void f2fs_drop_inmem_page(struct inode *inode, struct page *page) kmem_cache_free(inmem_entry_slab, cur); ClearPageUptodate(page); - set_page_private(page, 0); - ClearPagePrivate(page); + f2fs_clear_page_private(page); f2fs_put_page(page, 0); trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE); @@ -429,12 +442,15 @@ retry: * recovery or rewrite & commit last transaction. For other * error number, revoking was done by filesystem itself. */ - err = __revoke_inmem_pages(inode, &revoke_list, false, true); + err = __revoke_inmem_pages(inode, &revoke_list, + false, true, false); /* drop all uncommitted pages */ - __revoke_inmem_pages(inode, &fi->inmem_pages, true, false); + __revoke_inmem_pages(inode, &fi->inmem_pages, + true, false, false); } else { - __revoke_inmem_pages(inode, &revoke_list, false, false); + __revoke_inmem_pages(inode, &revoke_list, + false, false, false); } return err; @@ -542,9 +558,13 @@ void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) static int __submit_flush_wait(struct f2fs_sb_info *sbi, struct block_device *bdev) { - struct bio *bio = f2fs_bio_alloc(sbi, 0, true); + struct bio *bio; int ret; + bio = f2fs_bio_alloc(sbi, 0, false); + if (!bio) + return -ENOMEM; + bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH; bio_set_dev(bio, bdev); ret = submit_bio_wait(bio); @@ -868,6 +888,9 @@ int f2fs_disable_cp_again(struct f2fs_sb_info *sbi) if (holes[DATA] > ovp || holes[NODE] > ovp) return -EAGAIN; + if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) && + dirty_segments(sbi) > overprovision_segments(sbi)) + return -EAGAIN; return 0; } @@ -1037,6 +1060,7 @@ static void __init_discard_policy(struct f2fs_sb_info *sbi, dpolicy->max_requests = DEF_MAX_DISCARD_REQUEST; dpolicy->io_aware_gran = MAX_PLIST_NUM; + dpolicy->timeout = 0; if (discard_type == DPOLICY_BG) { dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME; @@ -1059,6 +1083,8 @@ static void __init_discard_policy(struct f2fs_sb_info *sbi, } else if (discard_type == DPOLICY_UMOUNT) { dpolicy->max_requests = UINT_MAX; dpolicy->io_aware = false; + /* we need to issue all to keep CP_TRIMMED_FLAG */ + dpolicy->granularity = 1; } } @@ -1424,7 +1450,14 @@ static int __issue_discard_cmd(struct f2fs_sb_info *sbi, int i, issued = 0; bool io_interrupted = false; + if (dpolicy->timeout != 0) + f2fs_update_time(sbi, dpolicy->timeout); + for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { + if (dpolicy->timeout != 0 && + f2fs_time_over(sbi, dpolicy->timeout)) + break; + if (i + 1 < dpolicy->granularity) break; @@ -1611,7 +1644,7 @@ void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) } /* This comes from f2fs_put_super */ -bool f2fs_wait_discard_bios(struct f2fs_sb_info *sbi) +bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) { struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; struct discard_policy dpolicy; @@ -1619,6 +1652,7 @@ bool f2fs_wait_discard_bios(struct f2fs_sb_info *sbi) __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, dcc->discard_granularity); + dpolicy.timeout = UMOUNT_DISCARD_TIMEOUT; __issue_discard_cmd(sbi, &dpolicy); dropped = __drop_discard_cmd(sbi); @@ -3164,10 +3198,10 @@ int f2fs_inplace_write_data(struct f2fs_io_info *fio) stat_inc_inplace_blocks(fio->sbi); err = f2fs_submit_page_bio(fio); - if (!err) + if (!err) { update_device_state(fio); - - f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); + f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); + } return err; } diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index a77f76f528b61..5c7ed0442d6e2 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -865,7 +865,7 @@ static inline void wake_up_discard_thread(struct f2fs_sb_info *sbi, bool force) } } mutex_unlock(&dcc->cmd_lock); - if (!wakeup) + if (!wakeup || !is_idle(sbi, DISCARD_TIME)) return; wake_up: dcc->discard_wake = 1; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index d1ccc52afc939..f2aaa2cc6b3e0 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -269,7 +269,7 @@ static int f2fs_set_qf_name(struct super_block *sb, int qtype, if (!qname) { f2fs_msg(sb, KERN_ERR, "Not enough memory for storing quotafile name"); - return -EINVAL; + return -ENOMEM; } if (F2FS_OPTION(sbi).s_qf_names[qtype]) { if (strcmp(F2FS_OPTION(sbi).s_qf_names[qtype], qname) == 0) @@ -586,7 +586,7 @@ static int parse_options(struct super_block *sb, char *options) case Opt_io_size_bits: if (args->from && match_int(args, &arg)) return -EINVAL; - if (arg > __ilog2_u32(BIO_MAX_PAGES)) { + if (arg <= 0 || arg > __ilog2_u32(BIO_MAX_PAGES)) { f2fs_msg(sb, KERN_WARNING, "Not support %d, larger than %d", 1 << arg, BIO_MAX_PAGES); @@ -821,6 +821,8 @@ static int parse_options(struct super_block *sb, char *options) } if (test_opt(sbi, INLINE_XATTR_SIZE)) { + int min_size, max_size; + if (!f2fs_sb_has_extra_attr(sbi) || !f2fs_sb_has_flexible_inline_xattr(sbi)) { f2fs_msg(sb, KERN_ERR, @@ -834,14 +836,15 @@ static int parse_options(struct super_block *sb, char *options) "set with inline_xattr option"); return -EINVAL; } - if (!F2FS_OPTION(sbi).inline_xattr_size || - F2FS_OPTION(sbi).inline_xattr_size >= - DEF_ADDRS_PER_INODE - - F2FS_TOTAL_EXTRA_ATTR_SIZE - - DEF_INLINE_RESERVED_SIZE - - DEF_MIN_INLINE_SIZE) { + + min_size = sizeof(struct f2fs_xattr_header) / sizeof(__le32); + max_size = MAX_INLINE_XATTR_SIZE; + + if (F2FS_OPTION(sbi).inline_xattr_size < min_size || + F2FS_OPTION(sbi).inline_xattr_size > max_size) { f2fs_msg(sb, KERN_ERR, - "inline xattr size is out of range"); + "inline xattr size is out of range: %d ~ %d", + min_size, max_size); return -EINVAL; } } @@ -915,6 +918,10 @@ static int f2fs_drop_inode(struct inode *inode) sb_start_intwrite(inode->i_sb); f2fs_i_size_write(inode, 0); + f2fs_submit_merged_write_cond(F2FS_I_SB(inode), + inode, NULL, 0, DATA); + truncate_inode_pages_final(inode->i_mapping); + if (F2FS_HAS_BLOCKS(inode)) f2fs_truncate(inode); @@ -1048,7 +1055,7 @@ static void f2fs_put_super(struct super_block *sb) } /* be sure to wait for any on-going discard commands */ - dropped = f2fs_wait_discard_bios(sbi); + dropped = f2fs_issue_discard_timeout(sbi); if ((f2fs_hw_support_discard(sbi) || f2fs_hw_should_discard(sbi)) && !sbi->discard_blks && !dropped) { @@ -1075,7 +1082,10 @@ static void f2fs_put_super(struct super_block *sb) f2fs_bug_on(sbi, sbi->fsync_node_num); iput(sbi->node_inode); + sbi->node_inode = NULL; + iput(sbi->meta_inode); + sbi->meta_inode = NULL; /* * iput() can update stat information, if f2fs_write_checkpoint() @@ -1455,9 +1465,16 @@ static int f2fs_enable_quotas(struct super_block *sb); static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) { + unsigned int s_flags = sbi->sb->s_flags; struct cp_control cpc; - int err; + int err = 0; + int ret; + if (s_flags & SB_RDONLY) { + f2fs_msg(sbi->sb, KERN_ERR, + "checkpoint=disable on readonly fs"); + return -EINVAL; + } sbi->sb->s_flags |= SB_ACTIVE; f2fs_update_time(sbi, DISABLE_TIME); @@ -1465,18 +1482,24 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) while (!f2fs_time_over(sbi, DISABLE_TIME)) { mutex_lock(&sbi->gc_mutex); err = f2fs_gc(sbi, true, false, NULL_SEGNO); - if (err == -ENODATA) + if (err == -ENODATA) { + err = 0; break; + } if (err && err != -EAGAIN) - return err; + break; } - err = sync_filesystem(sbi->sb); - if (err) - return err; + ret = sync_filesystem(sbi->sb); + if (ret || err) { + err = ret ? ret: err; + goto restore_flag; + } - if (f2fs_disable_cp_again(sbi)) - return -EAGAIN; + if (f2fs_disable_cp_again(sbi)) { + err = -EAGAIN; + goto restore_flag; + } mutex_lock(&sbi->gc_mutex); cpc.reason = CP_PAUSE; @@ -1485,7 +1508,9 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) sbi->unusable_block_count = 0; mutex_unlock(&sbi->gc_mutex); - return 0; +restore_flag: + sbi->sb->s_flags = s_flags; /* Restore MS_RDONLY status */ + return err; } static void f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) @@ -2023,6 +2048,12 @@ void f2fs_quota_off_umount(struct super_block *sb) set_sbi_flag(F2FS_SB(sb), SBI_QUOTA_NEED_REPAIR); } } + /* + * In case of checkpoint=disable, we must flush quota blocks. + * This can cause NULL exception for node_inode in end_io, since + * put_super already dropped it. + */ + sync_filesystem(sb); } static void f2fs_truncate_quota_inode_pages(struct super_block *sb) @@ -2703,6 +2734,8 @@ static void init_sb_info(struct f2fs_sb_info *sbi) sbi->interval_time[DISCARD_TIME] = DEF_IDLE_INTERVAL; sbi->interval_time[GC_TIME] = DEF_IDLE_INTERVAL; sbi->interval_time[DISABLE_TIME] = DEF_DISABLE_INTERVAL; + sbi->interval_time[UMOUNT_DISCARD_TIMEOUT] = + DEF_UMOUNT_DISCARD_TIMEOUT; clear_sbi_flag(sbi, SBI_NEED_FSCK); for (i = 0; i < NR_COUNT_TYPE; i++) @@ -3022,10 +3055,11 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent) struct f2fs_super_block *raw_super; struct inode *root; int err; - bool retry = true, need_fsck = false; + bool skip_recovery = false, need_fsck = false; char *options = NULL; int recovery, i, valid_super_block; struct curseg_info *seg_i; + int retry_cnt = 1; try_onemore: err = -EINVAL; @@ -3097,7 +3131,6 @@ try_onemore: sb->s_maxbytes = sbi->max_file_blocks << le32_to_cpu(raw_super->log_blocksize); sb->s_max_links = F2FS_LINK_MAX; - get_random_bytes(&sbi->s_next_generation, sizeof(u32)); #ifdef CONFIG_QUOTA sb->dq_op = &f2fs_quota_operations; @@ -3200,6 +3233,10 @@ try_onemore: if (__is_set_ckpt_flags(F2FS_CKPT(sbi), CP_QUOTA_NEED_FSCK_FLAG)) set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + if (__is_set_ckpt_flags(F2FS_CKPT(sbi), CP_DISABLED_QUICK_FLAG)) { + set_sbi_flag(sbi, SBI_CP_DISABLED_QUICK); + sbi->interval_time[DISABLE_TIME] = DEF_DISABLE_QUICK_INTERVAL; + } /* Initialize device list */ err = f2fs_scan_devices(sbi); @@ -3288,7 +3325,7 @@ try_onemore: sb->s_root = d_make_root(root); /* allocate root dentry */ if (!sb->s_root) { err = -ENOMEM; - goto free_root_inode; + goto free_node_inode; } err = f2fs_register_sysfs(sbi); @@ -3310,7 +3347,7 @@ try_onemore: goto free_meta; if (unlikely(is_set_ckpt_flags(sbi, CP_DISABLED_FLAG))) - goto skip_recovery; + goto reset_checkpoint; /* recover fsynced data */ if (!test_opt(sbi, DISABLE_ROLL_FORWARD)) { @@ -3327,11 +3364,13 @@ try_onemore: if (need_fsck) set_sbi_flag(sbi, SBI_NEED_FSCK); - if (!retry) - goto skip_recovery; + if (skip_recovery) + goto reset_checkpoint; err = f2fs_recover_fsync_data(sbi, false); if (err < 0) { + if (err != -ENOMEM) + skip_recovery = true; need_fsck = true; f2fs_msg(sb, KERN_ERR, "Cannot recover all fsync data errno=%d", err); @@ -3347,14 +3386,14 @@ try_onemore: goto free_meta; } } -skip_recovery: +reset_checkpoint: /* f2fs_recover_fsync_data() cleared this already */ clear_sbi_flag(sbi, SBI_POR_DOING); if (test_opt(sbi, DISABLE_CHECKPOINT)) { err = f2fs_disable_checkpoint(sbi); if (err) - goto free_meta; + goto sync_free_meta; } else if (is_set_ckpt_flags(sbi, CP_DISABLED_FLAG)) { f2fs_enable_checkpoint(sbi); } @@ -3367,7 +3406,7 @@ skip_recovery: /* After POR, we can run background GC thread.*/ err = f2fs_start_gc_thread(sbi); if (err) - goto free_meta; + goto sync_free_meta; } kvfree(options); @@ -3387,8 +3426,14 @@ skip_recovery: cur_cp_version(F2FS_CKPT(sbi))); f2fs_update_time(sbi, CP_TIME); f2fs_update_time(sbi, REQ_TIME); + clear_sbi_flag(sbi, SBI_CP_DISABLED_QUICK); return 0; +sync_free_meta: + /* safe to flush all the data */ + sync_filesystem(sbi->sb); + retry_cnt = 0; + free_meta: #ifdef CONFIG_QUOTA f2fs_truncate_quota_inode_pages(sb); @@ -3402,6 +3447,8 @@ free_meta: * falls into an infinite loop in f2fs_sync_meta_pages(). */ truncate_inode_pages_final(META_MAPPING(sbi)); + /* evict some inodes being cached by GC */ + evict_inodes(sb); f2fs_unregister_sysfs(sbi); free_root_inode: dput(sb->s_root); @@ -3410,6 +3457,7 @@ free_node_inode: f2fs_release_ino_entry(sbi, true); truncate_inode_pages_final(NODE_MAPPING(sbi)); iput(sbi->node_inode); + sbi->node_inode = NULL; free_stats: f2fs_destroy_stats(sbi); free_nm: @@ -3422,6 +3470,7 @@ free_devices: free_meta_inode: make_bad_inode(sbi->meta_inode); iput(sbi->meta_inode); + sbi->meta_inode = NULL; free_io_dummy: mempool_destroy(sbi->write_io_dummy); free_percpu: @@ -3443,8 +3492,8 @@ free_sbi: kvfree(sbi); /* give only one another chance */ - if (retry) { - retry = false; + if (retry_cnt > 0 && skip_recovery) { + retry_cnt--; shrink_dcache_sb(sb); goto try_onemore; } diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 70da6801c86f4..729f46a3c9ee0 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -222,6 +222,8 @@ out: #ifdef CONFIG_F2FS_FAULT_INJECTION if (a->struct_type == FAULT_INFO_TYPE && t >= (1 << FAULT_MAX)) return -EINVAL; + if (a->struct_type == FAULT_INFO_RATE && t >= UINT_MAX) + return -EINVAL; #endif if (a->struct_type == RESERVED_BLOCKS) { spin_lock(&sbi->stat_lock); @@ -278,10 +280,16 @@ out: return count; } - *ui = t; - if (!strcmp(a->attr.name, "iostat_enable") && *ui == 0) - f2fs_reset_iostat(sbi); + if (!strcmp(a->attr.name, "iostat_enable")) { + sbi->iostat_enable = !!t; + if (!sbi->iostat_enable) + f2fs_reset_iostat(sbi); + return count; + } + + *ui = (unsigned int)t; + return count; } @@ -418,6 +426,8 @@ F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, idle_interval, interval_time[REQ_TIME]); F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, discard_idle_interval, interval_time[DISCARD_TIME]); F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, gc_idle_interval, interval_time[GC_TIME]); +F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, + umount_discard_timeout, interval_time[UMOUNT_DISCARD_TIMEOUT]); F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, iostat_enable, iostat_enable); F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, readdir_ra, readdir_ra); F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, gc_pin_file_thresh, gc_pin_file_threshold); @@ -475,6 +485,7 @@ static struct attribute *f2fs_attrs[] = { ATTR_LIST(idle_interval), ATTR_LIST(discard_idle_interval), ATTR_LIST(gc_idle_interval), + ATTR_LIST(umount_discard_timeout), ATTR_LIST(iostat_enable), ATTR_LIST(readdir_ra), ATTR_LIST(gc_pin_file_thresh), diff --git a/fs/f2fs/trace.c b/fs/f2fs/trace.c index ce2a5eb210b66..d0ab533a9ce89 100644 --- a/fs/f2fs/trace.c +++ b/fs/f2fs/trace.c @@ -14,7 +14,7 @@ #include "trace.h" static RADIX_TREE(pids, GFP_ATOMIC); -static struct mutex pids_lock; +static spinlock_t pids_lock; static struct last_io_info last_io; static inline void __print_last_io(void) @@ -58,23 +58,29 @@ void f2fs_trace_pid(struct page *page) set_page_private(page, (unsigned long)pid); +retry: if (radix_tree_preload(GFP_NOFS)) return; - mutex_lock(&pids_lock); + spin_lock(&pids_lock); p = radix_tree_lookup(&pids, pid); if (p == current) goto out; if (p) radix_tree_delete(&pids, pid); - f2fs_radix_tree_insert(&pids, pid, current); + if (radix_tree_insert(&pids, pid, current)) { + spin_unlock(&pids_lock); + radix_tree_preload_end(); + cond_resched(); + goto retry; + } trace_printk("%3x:%3x %4x %-16s\n", MAJOR(inode->i_sb->s_dev), MINOR(inode->i_sb->s_dev), pid, current->comm); out: - mutex_unlock(&pids_lock); + spin_unlock(&pids_lock); radix_tree_preload_end(); } @@ -119,7 +125,7 @@ void f2fs_trace_ios(struct f2fs_io_info *fio, int flush) void f2fs_build_trace_ios(void) { - mutex_init(&pids_lock); + spin_lock_init(&pids_lock); } #define PIDVEC_SIZE 128 @@ -147,7 +153,7 @@ void f2fs_destroy_trace_ios(void) pid_t next_pid = 0; unsigned int found; - mutex_lock(&pids_lock); + spin_lock(&pids_lock); while ((found = gang_lookup_pids(pid, next_pid, PIDVEC_SIZE))) { unsigned idx; @@ -155,5 +161,5 @@ void f2fs_destroy_trace_ios(void) for (idx = 0; idx < found; idx++) radix_tree_delete(&pids, pid[idx]); } - mutex_unlock(&pids_lock); + spin_unlock(&pids_lock); } diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index 18d5ffbc5e8c6..848a785abe253 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -224,11 +224,11 @@ static struct f2fs_xattr_entry *__find_inline_xattr(struct inode *inode, { struct f2fs_xattr_entry *entry; unsigned int inline_size = inline_xattr_size(inode); + void *max_addr = base_addr + inline_size; list_for_each_xattr(entry, base_addr) { - if ((void *)entry + sizeof(__u32) > base_addr + inline_size || - (void *)XATTR_NEXT_ENTRY(entry) + sizeof(__u32) > - base_addr + inline_size) { + if ((void *)entry + sizeof(__u32) > max_addr || + (void *)XATTR_NEXT_ENTRY(entry) > max_addr) { *last_addr = entry; return NULL; } @@ -239,6 +239,13 @@ static struct f2fs_xattr_entry *__find_inline_xattr(struct inode *inode, if (!memcmp(entry->e_name, name, len)) break; } + + /* inline xattr header or entry across max inline xattr size */ + if (IS_XATTR_LAST_ENTRY(entry) && + (void *)entry + sizeof(__u32) > max_addr) { + *last_addr = entry; + return NULL; + } return entry; } @@ -340,7 +347,7 @@ check: *base_addr = txattr_addr; return 0; out: - kzfree(txattr_addr); + kvfree(txattr_addr); return err; } @@ -383,7 +390,7 @@ static int read_all_xattrs(struct inode *inode, struct page *ipage, *base_addr = txattr_addr; return 0; fail: - kzfree(txattr_addr); + kvfree(txattr_addr); return err; } @@ -510,7 +517,7 @@ int f2fs_getxattr(struct inode *inode, int index, const char *name, } error = size; out: - kzfree(base_addr); + kvfree(base_addr); return error; } @@ -538,7 +545,7 @@ ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) if (!handler || (handler->list && !handler->list(dentry))) continue; - prefix = handler->prefix ?: handler->name; + prefix = xattr_prefix(handler); prefix_len = strlen(prefix); size = prefix_len + entry->e_name_len + 1; if (buffer) { @@ -556,7 +563,7 @@ ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) } error = buffer_size - rest; cleanup: - kzfree(base_addr); + kvfree(base_addr); return error; } @@ -687,7 +694,7 @@ static int __f2fs_setxattr(struct inode *inode, int index, if (!error && S_ISDIR(inode->i_mode)) set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_CP); exit: - kzfree(base_addr); + kvfree(base_addr); return error; } diff --git a/fs/f2fs/xattr.h b/fs/f2fs/xattr.h index 67db134da0f5f..9172ee082ca83 100644 --- a/fs/f2fs/xattr.h +++ b/fs/f2fs/xattr.h @@ -78,6 +78,12 @@ struct f2fs_xattr_entry { sizeof(struct f2fs_xattr_header) - \ sizeof(struct f2fs_xattr_entry)) +#define MAX_INLINE_XATTR_SIZE \ + (DEF_ADDRS_PER_INODE - \ + F2FS_TOTAL_EXTRA_ATTR_SIZE / sizeof(__le32) - \ + DEF_INLINE_RESERVED_SIZE - \ + MIN_INLINE_DENTRY_SIZE / sizeof(__le32)) + /* * On-disk structure of f2fs_xattr * We use inline xattrs space + 1 block for xattr. diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 8247bd1634cb8..7066cd7c7aff3 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -1889,7 +1889,7 @@ lookup_again: atomic_read(&lo->plh_outstanding) != 0) { spin_unlock(&ino->i_lock); lseg = ERR_PTR(wait_var_event_killable(&lo->plh_outstanding, - atomic_read(&lo->plh_outstanding))); + !atomic_read(&lo->plh_outstanding))); if (IS_ERR(lseg) || !list_empty(&lo->plh_segs)) goto out_put_layout_hdr; pnfs_put_layout_hdr(lo); diff --git a/fs/proc/base.c b/fs/proc/base.c index f5ebdd87afb24..ddef482f13340 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -3074,6 +3074,15 @@ static const struct file_operations proc_tgid_base_operations = { .llseek = generic_file_llseek, }; +struct pid *tgid_pidfd_to_pid(const struct file *file) +{ + if (!d_is_dir(file->f_path.dentry) || + (file->f_op != &proc_tgid_base_operations)) + return ERR_PTR(-EBADF); + + return proc_pid(file_inode(file)); +} + static struct dentry *proc_tgid_base_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { return proc_pident_lookup(dir, dentry, diff --git a/fs/sysfs/mount.c b/fs/sysfs/mount.c index 4cb21b558a856..1b56686ab178d 100644 --- a/fs/sysfs/mount.c +++ b/fs/sysfs/mount.c @@ -71,9 +71,11 @@ static int sysfs_init_fs_context(struct fs_context *fc) kfc->magic = SYSFS_MAGIC; fc->fs_private = kfc; fc->ops = &sysfs_fs_context_ops; - if (fc->user_ns) - put_user_ns(fc->user_ns); - fc->user_ns = get_user_ns(netns->user_ns); + if (netns) { + if (fc->user_ns) + put_user_ns(fc->user_ns); + fc->user_ns = get_user_ns(netns->user_ns); + } fc->global = true; return 0; } diff --git a/fs/xfs/libxfs/xfs_dir2_leaf.c b/fs/xfs/libxfs/xfs_dir2_leaf.c index 9a3767818c507..9c2a0a13ed612 100644 --- a/fs/xfs/libxfs/xfs_dir2_leaf.c +++ b/fs/xfs/libxfs/xfs_dir2_leaf.c @@ -563,43 +563,40 @@ xfs_dir3_leaf_find_entry( */ int /* error */ xfs_dir2_leaf_addname( - xfs_da_args_t *args) /* operation arguments */ + struct xfs_da_args *args) /* operation arguments */ { + struct xfs_dir3_icleaf_hdr leafhdr; + struct xfs_trans *tp = args->trans; __be16 *bestsp; /* freespace table in leaf */ - int compact; /* need to compact leaves */ - xfs_dir2_data_hdr_t *hdr; /* data block header */ + __be16 *tagp; /* end of data entry */ struct xfs_buf *dbp; /* data block buffer */ - xfs_dir2_data_entry_t *dep; /* data block entry */ - xfs_inode_t *dp; /* incore directory inode */ - xfs_dir2_data_unused_t *dup; /* data unused entry */ + struct xfs_buf *lbp; /* leaf's buffer */ + struct xfs_dir2_leaf *leaf; /* leaf structure */ + struct xfs_inode *dp = args->dp; /* incore directory inode */ + struct xfs_dir2_data_hdr *hdr; /* data block header */ + struct xfs_dir2_data_entry *dep; /* data block entry */ + struct xfs_dir2_leaf_entry *lep; /* leaf entry table pointer */ + struct xfs_dir2_leaf_entry *ents; + struct xfs_dir2_data_unused *dup; /* data unused entry */ + struct xfs_dir2_leaf_tail *ltp; /* leaf tail pointer */ + struct xfs_dir2_data_free *bf; /* bestfree table */ + int compact; /* need to compact leaves */ int error; /* error return value */ int grown; /* allocated new data block */ - int highstale; /* index of next stale leaf */ + int highstale = 0; /* index of next stale leaf */ int i; /* temporary, index */ int index; /* leaf table position */ - struct xfs_buf *lbp; /* leaf's buffer */ - xfs_dir2_leaf_t *leaf; /* leaf structure */ int length; /* length of new entry */ - xfs_dir2_leaf_entry_t *lep; /* leaf entry table pointer */ int lfloglow; /* low leaf logging index */ int lfloghigh; /* high leaf logging index */ - int lowstale; /* index of prev stale leaf */ - xfs_dir2_leaf_tail_t *ltp; /* leaf tail pointer */ + int lowstale = 0; /* index of prev stale leaf */ int needbytes; /* leaf block bytes needed */ int needlog; /* need to log data header */ int needscan; /* need to rescan data free */ - __be16 *tagp; /* end of data entry */ - xfs_trans_t *tp; /* transaction pointer */ xfs_dir2_db_t use_block; /* data block number */ - struct xfs_dir2_data_free *bf; /* bestfree table */ - struct xfs_dir2_leaf_entry *ents; - struct xfs_dir3_icleaf_hdr leafhdr; trace_xfs_dir2_leaf_addname(args); - dp = args->dp; - tp = args->trans; - error = xfs_dir3_leaf_read(tp, dp, args->geo->leafblk, -1, &lbp); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_dir2_node.c b/fs/xfs/libxfs/xfs_dir2_node.c index 3b03703c5c3db..16731d2d684be 100644 --- a/fs/xfs/libxfs/xfs_dir2_node.c +++ b/fs/xfs/libxfs/xfs_dir2_node.c @@ -426,24 +426,22 @@ xfs_dir2_leaf_to_node( static int /* error */ xfs_dir2_leafn_add( struct xfs_buf *bp, /* leaf buffer */ - xfs_da_args_t *args, /* operation arguments */ + struct xfs_da_args *args, /* operation arguments */ int index) /* insertion pt for new entry */ { + struct xfs_dir3_icleaf_hdr leafhdr; + struct xfs_inode *dp = args->dp; + struct xfs_dir2_leaf *leaf = bp->b_addr; + struct xfs_dir2_leaf_entry *lep; + struct xfs_dir2_leaf_entry *ents; int compact; /* compacting stale leaves */ - xfs_inode_t *dp; /* incore directory inode */ - int highstale; /* next stale entry */ - xfs_dir2_leaf_t *leaf; /* leaf structure */ - xfs_dir2_leaf_entry_t *lep; /* leaf entry */ + int highstale = 0; /* next stale entry */ int lfloghigh; /* high leaf entry logging */ int lfloglow; /* low leaf entry logging */ - int lowstale; /* previous stale entry */ - struct xfs_dir3_icleaf_hdr leafhdr; - struct xfs_dir2_leaf_entry *ents; + int lowstale = 0; /* previous stale entry */ trace_xfs_dir2_leafn_add(args, index); - dp = args->dp; - leaf = bp->b_addr; dp->d_ops->leaf_hdr_from_disk(&leafhdr, leaf); ents = dp->d_ops->leaf_ents_p(leaf); diff --git a/include/clocksource/arm_arch_timer.h b/include/clocksource/arm_arch_timer.h index 349e5957c9498..702967d996bb3 100644 --- a/include/clocksource/arm_arch_timer.h +++ b/include/clocksource/arm_arch_timer.h @@ -74,6 +74,7 @@ enum arch_timer_spi_nr { struct arch_timer_kvm_info { struct timecounter timecounter; int virtual_irq; + int physical_irq; }; struct arch_timer_mem_frame { diff --git a/include/drm/drm_fb_helper.h b/include/drm/drm_fb_helper.h index bb9acea613693..286d58efed5d0 100644 --- a/include/drm/drm_fb_helper.h +++ b/include/drm/drm_fb_helper.h @@ -36,6 +36,7 @@ struct drm_fb_helper; #include <drm/drm_crtc.h> #include <drm/drm_device.h> #include <linux/kgdb.h> +#include <linux/vgaarb.h> enum mode_set_atomic { LEAVE_ATOMIC_MODE_SET, @@ -642,11 +643,18 @@ drm_fb_helper_remove_conflicting_pci_framebuffers(struct pci_dev *pdev, int resource_id, const char *name) { + int ret = 0; + + /* + * WARNING: Apparently we must kick fbdev drivers before vgacon, + * otherwise the vga fbdev driver falls over. + */ #if IS_REACHABLE(CONFIG_FB) - return remove_conflicting_pci_framebuffers(pdev, resource_id, name); -#else - return 0; + ret = remove_conflicting_pci_framebuffers(pdev, resource_id, name); #endif + if (ret == 0) + ret = vga_remove_vgacon(pdev); + return ret; } #endif diff --git a/include/dt-bindings/clock/actions,s500-cmu.h b/include/dt-bindings/clock/actions,s500-cmu.h new file mode 100644 index 0000000000000..030981cd2d563 --- /dev/null +++ b/include/dt-bindings/clock/actions,s500-cmu.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Device Tree binding constants for Actions Semi S500 Clock Management Unit + * + * Copyright (c) 2014 Actions Semi Inc. + * Copyright (c) 2018 LSI-TEC - Caninos Loucos + */ + +#ifndef __DT_BINDINGS_CLOCK_S500_CMU_H +#define __DT_BINDINGS_CLOCK_S500_CMU_H + +#define CLK_NONE 0 + +/* fixed rate clocks */ +#define CLK_LOSC 1 +#define CLK_HOSC 2 + +/* pll clocks */ +#define CLK_CORE_PLL 3 +#define CLK_DEV_PLL 4 +#define CLK_DDR_PLL 5 +#define CLK_NAND_PLL 6 +#define CLK_DISPLAY_PLL 7 +#define CLK_ETHERNET_PLL 8 +#define CLK_AUDIO_PLL 9 + +/* system clock */ +#define CLK_DEV 10 +#define CLK_H 11 +#define CLK_AHBPREDIV 12 +#define CLK_AHB 13 +#define CLK_DE 14 +#define CLK_BISP 15 +#define CLK_VCE 16 +#define CLK_VDE 17 + +/* peripheral device clock */ +#define CLK_TIMER 18 +#define CLK_I2C0 19 +#define CLK_I2C1 20 +#define CLK_I2C2 21 +#define CLK_I2C3 22 +#define CLK_PWM0 23 +#define CLK_PWM1 24 +#define CLK_PWM2 25 +#define CLK_PWM3 26 +#define CLK_PWM4 27 +#define CLK_PWM5 28 +#define CLK_SD0 29 +#define CLK_SD1 30 +#define CLK_SD2 31 +#define CLK_SENSOR0 32 +#define CLK_SENSOR1 33 +#define CLK_SPI0 34 +#define CLK_SPI1 35 +#define CLK_SPI2 36 +#define CLK_SPI3 37 +#define CLK_UART0 38 +#define CLK_UART1 39 +#define CLK_UART2 40 +#define CLK_UART3 41 +#define CLK_UART4 42 +#define CLK_UART5 43 +#define CLK_UART6 44 +#define CLK_DE1 45 +#define CLK_DE2 46 +#define CLK_I2SRX 47 +#define CLK_I2STX 48 +#define CLK_HDMI_AUDIO 49 +#define CLK_HDMI 50 +#define CLK_SPDIF 51 +#define CLK_NAND 52 +#define CLK_ECC 53 +#define CLK_RMII_REF 54 + +#define CLK_NR_CLKS (CLK_RMII_REF + 1) + +#endif /* __DT_BINDINGS_CLOCK_S500_CMU_H */ diff --git a/include/dt-bindings/clock/axg-aoclkc.h b/include/dt-bindings/clock/axg-aoclkc.h index 61955016a55b3..8ec4a269c7a6e 100644 --- a/include/dt-bindings/clock/axg-aoclkc.h +++ b/include/dt-bindings/clock/axg-aoclkc.h @@ -21,6 +21,11 @@ #define CLKID_AO_SAR_ADC_SEL 8 #define CLKID_AO_SAR_ADC_DIV 9 #define CLKID_AO_SAR_ADC_CLK 10 -#define CLKID_AO_ALT_XTAL 11 +#define CLKID_AO_CTS_OSCIN 11 +#define CLKID_AO_32K_PRE 12 +#define CLKID_AO_32K_DIV 13 +#define CLKID_AO_32K_SEL 14 +#define CLKID_AO_32K 15 +#define CLKID_AO_CTS_RTC_OSCIN 16 #endif diff --git a/include/dt-bindings/clock/exynos5433.h b/include/dt-bindings/clock/exynos5433.h index 98bd85ce1e452..25ffa53573a57 100644 --- a/include/dt-bindings/clock/exynos5433.h +++ b/include/dt-bindings/clock/exynos5433.h @@ -156,7 +156,7 @@ #define CLK_ACLK_G2D_266 220 #define CLK_ACLK_G2D_400 221 #define CLK_ACLK_G3D_400 222 -#define CLK_ACLK_IMEM_SSX_266 223 +#define CLK_ACLK_IMEM_SSSX_266 223 #define CLK_ACLK_BUS0_400 224 #define CLK_ACLK_BUS1_400 225 #define CLK_ACLK_IMEM_200 226 @@ -1406,4 +1406,10 @@ #define CAM1_NR_CLK 113 +/* CMU_IMEM */ +#define CLK_ACLK_SLIMSSS 2 +#define CLK_PCLK_SLIMSSS 35 + +#define IMEM_NR_CLK 36 + #endif /* _DT_BINDINGS_CLOCK_EXYNOS5433_H */ diff --git a/include/dt-bindings/clock/g12a-aoclkc.h b/include/dt-bindings/clock/g12a-aoclkc.h new file mode 100644 index 0000000000000..8db01ffbeb063 --- /dev/null +++ b/include/dt-bindings/clock/g12a-aoclkc.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright (c) 2016 BayLibre, SAS + * Author: Neil Armstrong <narmstrong@baylibre.com> + * + * Copyright (c) 2018 Amlogic, inc. + * Author: Qiufang Dai <qiufang.dai@amlogic.com> + */ + +#ifndef DT_BINDINGS_CLOCK_AMLOGIC_MESON_G12A_AOCLK +#define DT_BINDINGS_CLOCK_AMLOGIC_MESON_G12A_AOCLK + +#define CLKID_AO_AHB 0 +#define CLKID_AO_IR_IN 1 +#define CLKID_AO_I2C_M0 2 +#define CLKID_AO_I2C_S0 3 +#define CLKID_AO_UART 4 +#define CLKID_AO_PROD_I2C 5 +#define CLKID_AO_UART2 6 +#define CLKID_AO_IR_OUT 7 +#define CLKID_AO_SAR_ADC 8 +#define CLKID_AO_MAILBOX 9 +#define CLKID_AO_M3 10 +#define CLKID_AO_AHB_SRAM 11 +#define CLKID_AO_RTI 12 +#define CLKID_AO_M4_FCLK 13 +#define CLKID_AO_M4_HCLK 14 +#define CLKID_AO_CLK81 15 +#define CLKID_AO_SAR_ADC_CLK 18 +#define CLKID_AO_32K 23 +#define CLKID_AO_CEC 27 +#define CLKID_AO_CTS_RTC_OSCIN 28 + +#endif diff --git a/include/dt-bindings/clock/g12a-clkc.h b/include/dt-bindings/clock/g12a-clkc.h new file mode 100644 index 0000000000000..83b657038d1ea --- /dev/null +++ b/include/dt-bindings/clock/g12a-clkc.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR MIT */ +/* + * Meson-G12A clock tree IDs + * + * Copyright (c) 2018 Amlogic, Inc. All rights reserved. + */ + +#ifndef __G12A_CLKC_H +#define __G12A_CLKC_H + +#define CLKID_SYS_PLL 0 +#define CLKID_FIXED_PLL 1 +#define CLKID_FCLK_DIV2 2 +#define CLKID_FCLK_DIV3 3 +#define CLKID_FCLK_DIV4 4 +#define CLKID_FCLK_DIV5 5 +#define CLKID_FCLK_DIV7 6 +#define CLKID_GP0_PLL 7 +#define CLKID_CLK81 10 +#define CLKID_MPLL0 11 +#define CLKID_MPLL1 12 +#define CLKID_MPLL2 13 +#define CLKID_MPLL3 14 +#define CLKID_DDR 15 +#define CLKID_DOS 16 +#define CLKID_AUDIO_LOCKER 17 +#define CLKID_MIPI_DSI_HOST 18 +#define CLKID_ETH_PHY 19 +#define CLKID_ISA 20 +#define CLKID_PL301 21 +#define CLKID_PERIPHS 22 +#define CLKID_SPICC0 23 +#define CLKID_I2C 24 +#define CLKID_SANA 25 +#define CLKID_SD 26 +#define CLKID_RNG0 27 +#define CLKID_UART0 28 +#define CLKID_SPICC1 29 +#define CLKID_HIU_IFACE 30 +#define CLKID_MIPI_DSI_PHY 31 +#define CLKID_ASSIST_MISC 32 +#define CLKID_SD_EMMC_A 33 +#define CLKID_SD_EMMC_B 34 +#define CLKID_SD_EMMC_C 35 +#define CLKID_AUDIO_CODEC 36 +#define CLKID_AUDIO 37 +#define CLKID_ETH 38 +#define CLKID_DEMUX 39 +#define CLKID_AUDIO_IFIFO 40 +#define CLKID_ADC 41 +#define CLKID_UART1 42 +#define CLKID_G2D 43 +#define CLKID_RESET 44 +#define CLKID_PCIE_COMB 45 +#define CLKID_PARSER 46 +#define CLKID_USB 47 +#define CLKID_PCIE_PHY 48 +#define CLKID_AHB_ARB0 49 +#define CLKID_AHB_DATA_BUS 50 +#define CLKID_AHB_CTRL_BUS 51 +#define CLKID_HTX_HDCP22 52 +#define CLKID_HTX_PCLK 53 +#define CLKID_BT656 54 +#define CLKID_USB1_DDR_BRIDGE 55 +#define CLKID_MMC_PCLK 56 +#define CLKID_UART2 57 +#define CLKID_VPU_INTR 58 +#define CLKID_GIC 59 +#define CLKID_SD_EMMC_A_CLK0 60 +#define CLKID_SD_EMMC_B_CLK0 61 +#define CLKID_SD_EMMC_C_CLK0 62 +#define CLKID_HIFI_PLL 74 +#define CLKID_VCLK2_VENCI0 80 +#define CLKID_VCLK2_VENCI1 81 +#define CLKID_VCLK2_VENCP0 82 +#define CLKID_VCLK2_VENCP1 83 +#define CLKID_VCLK2_VENCT0 84 +#define CLKID_VCLK2_VENCT1 85 +#define CLKID_VCLK2_OTHER 86 +#define CLKID_VCLK2_ENCI 87 +#define CLKID_VCLK2_ENCP 88 +#define CLKID_DAC_CLK 89 +#define CLKID_AOCLK 90 +#define CLKID_IEC958 91 +#define CLKID_ENC480P 92 +#define CLKID_RNG1 93 +#define CLKID_VCLK2_ENCT 94 +#define CLKID_VCLK2_ENCL 95 +#define CLKID_VCLK2_VENCLMMC 96 +#define CLKID_VCLK2_VENCL 97 +#define CLKID_VCLK2_OTHER1 98 +#define CLKID_FCLK_DIV2P5 99 +#define CLKID_DMA 105 +#define CLKID_EFUSE 106 +#define CLKID_ROM_BOOT 107 +#define CLKID_RESET_SEC 108 +#define CLKID_SEC_AHB_APB3 109 +#define CLKID_VPU_0_SEL 110 +#define CLKID_VPU_0 112 +#define CLKID_VPU_1_SEL 113 +#define CLKID_VPU_1 115 +#define CLKID_VPU 116 +#define CLKID_VAPB_0_SEL 117 +#define CLKID_VAPB_0 119 +#define CLKID_VAPB_1_SEL 120 +#define CLKID_VAPB_1 122 +#define CLKID_VAPB_SEL 123 +#define CLKID_VAPB 124 +#define CLKID_HDMI_PLL 128 +#define CLKID_VID_PLL 129 +#define CLKID_VCLK 138 +#define CLKID_VCLK2 139 +#define CLKID_VCLK_DIV1 148 +#define CLKID_VCLK_DIV2 149 +#define CLKID_VCLK_DIV4 150 +#define CLKID_VCLK_DIV6 151 +#define CLKID_VCLK_DIV12 152 +#define CLKID_VCLK2_DIV1 153 +#define CLKID_VCLK2_DIV2 154 +#define CLKID_VCLK2_DIV4 155 +#define CLKID_VCLK2_DIV6 156 +#define CLKID_VCLK2_DIV12 157 +#define CLKID_CTS_ENCI 162 +#define CLKID_CTS_ENCP 163 +#define CLKID_CTS_VDAC 164 +#define CLKID_HDMI_TX 165 +#define CLKID_HDMI 168 +#define CLKID_MALI_0_SEL 169 +#define CLKID_MALI_0 171 +#define CLKID_MALI_1_SEL 172 +#define CLKID_MALI_1 174 +#define CLKID_MALI 175 +#define CLKID_MPLL_5OM 177 + +#endif /* __G12A_CLKC_H */ diff --git a/include/dt-bindings/clock/gxbb-aoclkc.h b/include/dt-bindings/clock/gxbb-aoclkc.h index 9d15e2221fdb7..ec3b26319fc49 100644 --- a/include/dt-bindings/clock/gxbb-aoclkc.h +++ b/include/dt-bindings/clock/gxbb-aoclkc.h @@ -63,5 +63,12 @@ #define CLKID_AO_UART2 4 #define CLKID_AO_IR_BLASTER 5 #define CLKID_AO_CEC_32K 6 +#define CLKID_AO_CTS_OSCIN 7 +#define CLKID_AO_32K_PRE 8 +#define CLKID_AO_32K_DIV 9 +#define CLKID_AO_32K_SEL 10 +#define CLKID_AO_32K 11 +#define CLKID_AO_CTS_RTC_OSCIN 12 +#define CLKID_AO_CLK81 13 #endif diff --git a/include/dt-bindings/clock/imx5-clock.h b/include/dt-bindings/clock/imx5-clock.h index d382fc71aa837..a81be5be67000 100644 --- a/include/dt-bindings/clock/imx5-clock.h +++ b/include/dt-bindings/clock/imx5-clock.h @@ -214,6 +214,7 @@ #define IMX5_CLK_IEEE1588_SEL 202 #define IMX5_CLK_IEEE1588_PODF 203 #define IMX5_CLK_IEEE1588_GATE 204 -#define IMX5_CLK_END 205 +#define IMX5_CLK_SCC2_IPG_GATE 205 +#define IMX5_CLK_END 206 #endif /* __DT_BINDINGS_CLOCK_IMX5_H */ diff --git a/include/dt-bindings/clock/imx8mm-clock.h b/include/dt-bindings/clock/imx8mm-clock.h new file mode 100644 index 0000000000000..1b4353e7b4862 --- /dev/null +++ b/include/dt-bindings/clock/imx8mm-clock.h @@ -0,0 +1,244 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2017-2018 NXP + */ + +#ifndef __DT_BINDINGS_CLOCK_IMX8MM_H +#define __DT_BINDINGS_CLOCK_IMX8MM_H + +#define IMX8MM_CLK_DUMMY 0 +#define IMX8MM_CLK_32K 1 +#define IMX8MM_CLK_24M 2 +#define IMX8MM_OSC_HDMI_CLK 3 +#define IMX8MM_CLK_EXT1 4 +#define IMX8MM_CLK_EXT2 5 +#define IMX8MM_CLK_EXT3 6 +#define IMX8MM_CLK_EXT4 7 +#define IMX8MM_AUDIO_PLL1_REF_SEL 8 +#define IMX8MM_AUDIO_PLL2_REF_SEL 9 +#define IMX8MM_VIDEO_PLL1_REF_SEL 10 +#define IMX8MM_DRAM_PLL_REF_SEL 11 +#define IMX8MM_GPU_PLL_REF_SEL 12 +#define IMX8MM_VPU_PLL_REF_SEL 13 +#define IMX8MM_ARM_PLL_REF_SEL 14 +#define IMX8MM_SYS_PLL1_REF_SEL 15 +#define IMX8MM_SYS_PLL2_REF_SEL 16 +#define IMX8MM_SYS_PLL3_REF_SEL 17 +#define IMX8MM_AUDIO_PLL1 18 +#define IMX8MM_AUDIO_PLL2 19 +#define IMX8MM_VIDEO_PLL1 20 +#define IMX8MM_DRAM_PLL 21 +#define IMX8MM_GPU_PLL 22 +#define IMX8MM_VPU_PLL 23 +#define IMX8MM_ARM_PLL 24 +#define IMX8MM_SYS_PLL1 25 +#define IMX8MM_SYS_PLL2 26 +#define IMX8MM_SYS_PLL3 27 +#define IMX8MM_AUDIO_PLL1_BYPASS 28 +#define IMX8MM_AUDIO_PLL2_BYPASS 29 +#define IMX8MM_VIDEO_PLL1_BYPASS 30 +#define IMX8MM_DRAM_PLL_BYPASS 31 +#define IMX8MM_GPU_PLL_BYPASS 32 +#define IMX8MM_VPU_PLL_BYPASS 33 +#define IMX8MM_ARM_PLL_BYPASS 34 +#define IMX8MM_SYS_PLL1_BYPASS 35 +#define IMX8MM_SYS_PLL2_BYPASS 36 +#define IMX8MM_SYS_PLL3_BYPASS 37 +#define IMX8MM_AUDIO_PLL1_OUT 38 +#define IMX8MM_AUDIO_PLL2_OUT 39 +#define IMX8MM_VIDEO_PLL1_OUT 40 +#define IMX8MM_DRAM_PLL_OUT 41 +#define IMX8MM_GPU_PLL_OUT 42 +#define IMX8MM_VPU_PLL_OUT 43 +#define IMX8MM_ARM_PLL_OUT 44 +#define IMX8MM_SYS_PLL1_OUT 45 +#define IMX8MM_SYS_PLL2_OUT 46 +#define IMX8MM_SYS_PLL3_OUT 47 +#define IMX8MM_SYS_PLL1_40M 48 +#define IMX8MM_SYS_PLL1_80M 49 +#define IMX8MM_SYS_PLL1_100M 50 +#define IMX8MM_SYS_PLL1_133M 51 +#define IMX8MM_SYS_PLL1_160M 52 +#define IMX8MM_SYS_PLL1_200M 53 +#define IMX8MM_SYS_PLL1_266M 54 +#define IMX8MM_SYS_PLL1_400M 55 +#define IMX8MM_SYS_PLL1_800M 56 +#define IMX8MM_SYS_PLL2_50M 57 +#define IMX8MM_SYS_PLL2_100M 58 +#define IMX8MM_SYS_PLL2_125M 59 +#define IMX8MM_SYS_PLL2_166M 60 +#define IMX8MM_SYS_PLL2_200M 61 +#define IMX8MM_SYS_PLL2_250M 62 +#define IMX8MM_SYS_PLL2_333M 63 +#define IMX8MM_SYS_PLL2_500M 64 +#define IMX8MM_SYS_PLL2_1000M 65 + +/* core */ +#define IMX8MM_CLK_A53_SRC 66 +#define IMX8MM_CLK_M4_SRC 67 +#define IMX8MM_CLK_VPU_SRC 68 +#define IMX8MM_CLK_GPU3D_SRC 69 +#define IMX8MM_CLK_GPU2D_SRC 70 +#define IMX8MM_CLK_A53_CG 71 +#define IMX8MM_CLK_M4_CG 72 +#define IMX8MM_CLK_VPU_CG 73 +#define IMX8MM_CLK_GPU3D_CG 74 +#define IMX8MM_CLK_GPU2D_CG 75 +#define IMX8MM_CLK_A53_DIV 76 +#define IMX8MM_CLK_M4_DIV 77 +#define IMX8MM_CLK_VPU_DIV 78 +#define IMX8MM_CLK_GPU3D_DIV 79 +#define IMX8MM_CLK_GPU2D_DIV 80 + +/* bus */ +#define IMX8MM_CLK_MAIN_AXI 81 +#define IMX8MM_CLK_ENET_AXI 82 +#define IMX8MM_CLK_NAND_USDHC_BUS 83 +#define IMX8MM_CLK_VPU_BUS 84 +#define IMX8MM_CLK_DISP_AXI 85 +#define IMX8MM_CLK_DISP_APB 86 +#define IMX8MM_CLK_DISP_RTRM 87 +#define IMX8MM_CLK_USB_BUS 88 +#define IMX8MM_CLK_GPU_AXI 89 +#define IMX8MM_CLK_GPU_AHB 90 +#define IMX8MM_CLK_NOC 91 +#define IMX8MM_CLK_NOC_APB 92 + +#define IMX8MM_CLK_AHB 93 +#define IMX8MM_CLK_AUDIO_AHB 94 +#define IMX8MM_CLK_IPG_ROOT 95 +#define IMX8MM_CLK_IPG_AUDIO_ROOT 96 + +#define IMX8MM_CLK_DRAM_ALT 97 +#define IMX8MM_CLK_DRAM_APB 98 +#define IMX8MM_CLK_VPU_G1 99 +#define IMX8MM_CLK_VPU_G2 100 +#define IMX8MM_CLK_DISP_DTRC 101 +#define IMX8MM_CLK_DISP_DC8000 102 +#define IMX8MM_CLK_PCIE1_CTRL 103 +#define IMX8MM_CLK_PCIE1_PHY 104 +#define IMX8MM_CLK_PCIE1_AUX 105 +#define IMX8MM_CLK_DC_PIXEL 106 +#define IMX8MM_CLK_LCDIF_PIXEL 107 +#define IMX8MM_CLK_SAI1 108 +#define IMX8MM_CLK_SAI2 109 +#define IMX8MM_CLK_SAI3 110 +#define IMX8MM_CLK_SAI4 111 +#define IMX8MM_CLK_SAI5 112 +#define IMX8MM_CLK_SAI6 113 +#define IMX8MM_CLK_SPDIF1 114 +#define IMX8MM_CLK_SPDIF2 115 +#define IMX8MM_CLK_ENET_REF 116 +#define IMX8MM_CLK_ENET_TIMER 117 +#define IMX8MM_CLK_ENET_PHY_REF 118 +#define IMX8MM_CLK_NAND 119 +#define IMX8MM_CLK_QSPI 120 +#define IMX8MM_CLK_USDHC1 121 +#define IMX8MM_CLK_USDHC2 122 +#define IMX8MM_CLK_I2C1 123 +#define IMX8MM_CLK_I2C2 124 +#define IMX8MM_CLK_I2C3 125 +#define IMX8MM_CLK_I2C4 126 +#define IMX8MM_CLK_UART1 127 +#define IMX8MM_CLK_UART2 128 +#define IMX8MM_CLK_UART3 129 +#define IMX8MM_CLK_UART4 130 +#define IMX8MM_CLK_USB_CORE_REF 131 +#define IMX8MM_CLK_USB_PHY_REF 132 +#define IMX8MM_CLK_ECSPI1 133 +#define IMX8MM_CLK_ECSPI2 134 +#define IMX8MM_CLK_PWM1 135 +#define IMX8MM_CLK_PWM2 136 +#define IMX8MM_CLK_PWM3 137 +#define IMX8MM_CLK_PWM4 138 +#define IMX8MM_CLK_GPT1 139 +#define IMX8MM_CLK_WDOG 140 +#define IMX8MM_CLK_WRCLK 141 +#define IMX8MM_CLK_DSI_CORE 142 +#define IMX8MM_CLK_DSI_PHY_REF 143 +#define IMX8MM_CLK_DSI_DBI 144 +#define IMX8MM_CLK_USDHC3 145 +#define IMX8MM_CLK_CSI1_CORE 146 +#define IMX8MM_CLK_CSI1_PHY_REF 147 +#define IMX8MM_CLK_CSI1_ESC 148 +#define IMX8MM_CLK_CSI2_CORE 149 +#define IMX8MM_CLK_CSI2_PHY_REF 150 +#define IMX8MM_CLK_CSI2_ESC 151 +#define IMX8MM_CLK_PCIE2_CTRL 152 +#define IMX8MM_CLK_PCIE2_PHY 153 +#define IMX8MM_CLK_PCIE2_AUX 154 +#define IMX8MM_CLK_ECSPI3 155 +#define IMX8MM_CLK_PDM 156 +#define IMX8MM_CLK_VPU_H1 157 +#define IMX8MM_CLK_CLKO1 158 + +#define IMX8MM_CLK_ECSPI1_ROOT 159 +#define IMX8MM_CLK_ECSPI2_ROOT 160 +#define IMX8MM_CLK_ECSPI3_ROOT 161 +#define IMX8MM_CLK_ENET1_ROOT 162 +#define IMX8MM_CLK_GPT1_ROOT 163 +#define IMX8MM_CLK_I2C1_ROOT 164 +#define IMX8MM_CLK_I2C2_ROOT 165 +#define IMX8MM_CLK_I2C3_ROOT 166 +#define IMX8MM_CLK_I2C4_ROOT 167 +#define IMX8MM_CLK_OCOTP_ROOT 168 +#define IMX8MM_CLK_PCIE1_ROOT 169 +#define IMX8MM_CLK_PWM1_ROOT 170 +#define IMX8MM_CLK_PWM2_ROOT 171 +#define IMX8MM_CLK_PWM3_ROOT 172 +#define IMX8MM_CLK_PWM4_ROOT 173 +#define IMX8MM_CLK_QSPI_ROOT 174 +#define IMX8MM_CLK_NAND_ROOT 175 +#define IMX8MM_CLK_SAI1_ROOT 176 +#define IMX8MM_CLK_SAI1_IPG 177 +#define IMX8MM_CLK_SAI2_ROOT 178 +#define IMX8MM_CLK_SAI2_IPG 179 +#define IMX8MM_CLK_SAI3_ROOT 180 +#define IMX8MM_CLK_SAI3_IPG 181 +#define IMX8MM_CLK_SAI4_ROOT 182 +#define IMX8MM_CLK_SAI4_IPG 183 +#define IMX8MM_CLK_SAI5_ROOT 184 +#define IMX8MM_CLK_SAI5_IPG 185 +#define IMX8MM_CLK_SAI6_ROOT 186 +#define IMX8MM_CLK_SAI6_IPG 187 +#define IMX8MM_CLK_UART1_ROOT 188 +#define IMX8MM_CLK_UART2_ROOT 189 +#define IMX8MM_CLK_UART3_ROOT 190 +#define IMX8MM_CLK_UART4_ROOT 191 +#define IMX8MM_CLK_USB1_CTRL_ROOT 192 +#define IMX8MM_CLK_GPU3D_ROOT 193 +#define IMX8MM_CLK_USDHC1_ROOT 194 +#define IMX8MM_CLK_USDHC2_ROOT 195 +#define IMX8MM_CLK_WDOG1_ROOT 196 +#define IMX8MM_CLK_WDOG2_ROOT 197 +#define IMX8MM_CLK_WDOG3_ROOT 198 +#define IMX8MM_CLK_VPU_G1_ROOT 199 +#define IMX8MM_CLK_GPU_BUS_ROOT 200 +#define IMX8MM_CLK_VPU_H1_ROOT 201 +#define IMX8MM_CLK_VPU_G2_ROOT 202 +#define IMX8MM_CLK_PDM_ROOT 203 +#define IMX8MM_CLK_DISP_ROOT 204 +#define IMX8MM_CLK_DISP_AXI_ROOT 205 +#define IMX8MM_CLK_DISP_APB_ROOT 206 +#define IMX8MM_CLK_DISP_RTRM_ROOT 207 +#define IMX8MM_CLK_USDHC3_ROOT 208 +#define IMX8MM_CLK_TMU_ROOT 209 +#define IMX8MM_CLK_VPU_DEC_ROOT 210 +#define IMX8MM_CLK_SDMA1_ROOT 211 +#define IMX8MM_CLK_SDMA2_ROOT 212 +#define IMX8MM_CLK_SDMA3_ROOT 213 +#define IMX8MM_CLK_GPT_3M 214 +#define IMX8MM_CLK_ARM 215 +#define IMX8MM_CLK_PDM_IPG 216 +#define IMX8MM_CLK_GPU2D_ROOT 217 +#define IMX8MM_CLK_MU_ROOT 218 +#define IMX8MM_CLK_CSI1_ROOT 219 + +#define IMX8MM_CLK_DRAM_CORE 220 +#define IMX8MM_CLK_DRAM_ALT_ROOT 221 + +#define IMX8MM_CLK_NAND_USDHC_BUS_RAWNAND_CLK 222 + +#define IMX8MM_CLK_END 223 + +#endif diff --git a/include/dt-bindings/clock/imx8mq-clock.h b/include/dt-bindings/clock/imx8mq-clock.h index 04f7ac345984f..6677e920dc2da 100644 --- a/include/dt-bindings/clock/imx8mq-clock.h +++ b/include/dt-bindings/clock/imx8mq-clock.h @@ -245,151 +245,160 @@ /* USB_CORE_REF */ #define IMX8MQ_CLK_USB_CORE_REF 152 /* USB_PHY_REF */ -#define IMX8MQ_CLK_USB_PHY_REF 163 +#define IMX8MQ_CLK_USB_PHY_REF 153 /* ECSPI1 */ -#define IMX8MQ_CLK_ECSPI1 164 +#define IMX8MQ_CLK_ECSPI1 154 /* ECSPI2 */ -#define IMX8MQ_CLK_ECSPI2 165 +#define IMX8MQ_CLK_ECSPI2 155 /* PWM1 */ -#define IMX8MQ_CLK_PWM1 166 +#define IMX8MQ_CLK_PWM1 156 /* PWM2 */ -#define IMX8MQ_CLK_PWM2 167 +#define IMX8MQ_CLK_PWM2 157 /* PWM3 */ -#define IMX8MQ_CLK_PWM3 168 +#define IMX8MQ_CLK_PWM3 158 /* PWM4 */ -#define IMX8MQ_CLK_PWM4 169 +#define IMX8MQ_CLK_PWM4 159 /* GPT1 */ -#define IMX8MQ_CLK_GPT1 170 +#define IMX8MQ_CLK_GPT1 160 /* WDOG */ -#define IMX8MQ_CLK_WDOG 171 +#define IMX8MQ_CLK_WDOG 161 /* WRCLK */ -#define IMX8MQ_CLK_WRCLK 172 +#define IMX8MQ_CLK_WRCLK 162 /* DSI_CORE */ -#define IMX8MQ_CLK_DSI_CORE 173 +#define IMX8MQ_CLK_DSI_CORE 163 /* DSI_PHY */ -#define IMX8MQ_CLK_DSI_PHY_REF 174 +#define IMX8MQ_CLK_DSI_PHY_REF 164 /* DSI_DBI */ -#define IMX8MQ_CLK_DSI_DBI 175 +#define IMX8MQ_CLK_DSI_DBI 165 /*DSI_ESC */ -#define IMX8MQ_CLK_DSI_ESC 176 +#define IMX8MQ_CLK_DSI_ESC 166 /* CSI1_CORE */ -#define IMX8MQ_CLK_CSI1_CORE 177 +#define IMX8MQ_CLK_CSI1_CORE 167 /* CSI1_PHY */ -#define IMX8MQ_CLK_CSI1_PHY_REF 178 +#define IMX8MQ_CLK_CSI1_PHY_REF 168 /* CSI_ESC */ -#define IMX8MQ_CLK_CSI1_ESC 179 +#define IMX8MQ_CLK_CSI1_ESC 169 /* CSI2_CORE */ #define IMX8MQ_CLK_CSI2_CORE 170 /* CSI2_PHY */ -#define IMX8MQ_CLK_CSI2_PHY_REF 181 +#define IMX8MQ_CLK_CSI2_PHY_REF 171 /* CSI2_ESC */ -#define IMX8MQ_CLK_CSI2_ESC 182 +#define IMX8MQ_CLK_CSI2_ESC 172 /* PCIE2_CTRL */ -#define IMX8MQ_CLK_PCIE2_CTRL 183 +#define IMX8MQ_CLK_PCIE2_CTRL 173 /* PCIE2_PHY */ -#define IMX8MQ_CLK_PCIE2_PHY 184 +#define IMX8MQ_CLK_PCIE2_PHY 174 /* PCIE2_AUX */ -#define IMX8MQ_CLK_PCIE2_AUX 185 +#define IMX8MQ_CLK_PCIE2_AUX 175 /* ECSPI3 */ -#define IMX8MQ_CLK_ECSPI3 186 +#define IMX8MQ_CLK_ECSPI3 176 /* CCGR clocks */ -#define IMX8MQ_CLK_A53_ROOT 187 -#define IMX8MQ_CLK_DRAM_ROOT 188 -#define IMX8MQ_CLK_ECSPI1_ROOT 189 +#define IMX8MQ_CLK_A53_ROOT 177 +#define IMX8MQ_CLK_DRAM_ROOT 178 +#define IMX8MQ_CLK_ECSPI1_ROOT 179 #define IMX8MQ_CLK_ECSPI2_ROOT 180 #define IMX8MQ_CLK_ECSPI3_ROOT 181 #define IMX8MQ_CLK_ENET1_ROOT 182 -#define IMX8MQ_CLK_GPT1_ROOT 193 -#define IMX8MQ_CLK_I2C1_ROOT 194 -#define IMX8MQ_CLK_I2C2_ROOT 195 -#define IMX8MQ_CLK_I2C3_ROOT 196 -#define IMX8MQ_CLK_I2C4_ROOT 197 -#define IMX8MQ_CLK_M4_ROOT 198 -#define IMX8MQ_CLK_PCIE1_ROOT 199 -#define IMX8MQ_CLK_PCIE2_ROOT 200 -#define IMX8MQ_CLK_PWM1_ROOT 201 -#define IMX8MQ_CLK_PWM2_ROOT 202 -#define IMX8MQ_CLK_PWM3_ROOT 203 -#define IMX8MQ_CLK_PWM4_ROOT 204 -#define IMX8MQ_CLK_QSPI_ROOT 205 -#define IMX8MQ_CLK_SAI1_ROOT 206 -#define IMX8MQ_CLK_SAI2_ROOT 207 -#define IMX8MQ_CLK_SAI3_ROOT 208 -#define IMX8MQ_CLK_SAI4_ROOT 209 -#define IMX8MQ_CLK_SAI5_ROOT 210 -#define IMX8MQ_CLK_SAI6_ROOT 212 -#define IMX8MQ_CLK_UART1_ROOT 213 -#define IMX8MQ_CLK_UART2_ROOT 214 -#define IMX8MQ_CLK_UART3_ROOT 215 -#define IMX8MQ_CLK_UART4_ROOT 216 -#define IMX8MQ_CLK_USB1_CTRL_ROOT 217 -#define IMX8MQ_CLK_USB2_CTRL_ROOT 218 -#define IMX8MQ_CLK_USB1_PHY_ROOT 219 -#define IMX8MQ_CLK_USB2_PHY_ROOT 220 -#define IMX8MQ_CLK_USDHC1_ROOT 221 -#define IMX8MQ_CLK_USDHC2_ROOT 222 -#define IMX8MQ_CLK_WDOG1_ROOT 223 -#define IMX8MQ_CLK_WDOG2_ROOT 224 -#define IMX8MQ_CLK_WDOG3_ROOT 225 -#define IMX8MQ_CLK_GPU_ROOT 226 -#define IMX8MQ_CLK_HEVC_ROOT 227 -#define IMX8MQ_CLK_AVC_ROOT 228 -#define IMX8MQ_CLK_VP9_ROOT 229 -#define IMX8MQ_CLK_HEVC_INTER_ROOT 230 -#define IMX8MQ_CLK_DISP_ROOT 231 -#define IMX8MQ_CLK_HDMI_ROOT 232 -#define IMX8MQ_CLK_HDMI_PHY_ROOT 233 -#define IMX8MQ_CLK_VPU_DEC_ROOT 234 -#define IMX8MQ_CLK_CSI1_ROOT 235 -#define IMX8MQ_CLK_CSI2_ROOT 236 -#define IMX8MQ_CLK_RAWNAND_ROOT 237 -#define IMX8MQ_CLK_SDMA1_ROOT 238 -#define IMX8MQ_CLK_SDMA2_ROOT 239 -#define IMX8MQ_CLK_VPU_G1_ROOT 240 -#define IMX8MQ_CLK_VPU_G2_ROOT 241 +#define IMX8MQ_CLK_GPT1_ROOT 183 +#define IMX8MQ_CLK_I2C1_ROOT 184 +#define IMX8MQ_CLK_I2C2_ROOT 185 +#define IMX8MQ_CLK_I2C3_ROOT 186 +#define IMX8MQ_CLK_I2C4_ROOT 187 +#define IMX8MQ_CLK_M4_ROOT 188 +#define IMX8MQ_CLK_PCIE1_ROOT 189 +#define IMX8MQ_CLK_PCIE2_ROOT 190 +#define IMX8MQ_CLK_PWM1_ROOT 191 +#define IMX8MQ_CLK_PWM2_ROOT 192 +#define IMX8MQ_CLK_PWM3_ROOT 193 +#define IMX8MQ_CLK_PWM4_ROOT 194 +#define IMX8MQ_CLK_QSPI_ROOT 195 +#define IMX8MQ_CLK_SAI1_ROOT 196 +#define IMX8MQ_CLK_SAI2_ROOT 197 +#define IMX8MQ_CLK_SAI3_ROOT 198 +#define IMX8MQ_CLK_SAI4_ROOT 199 +#define IMX8MQ_CLK_SAI5_ROOT 200 +#define IMX8MQ_CLK_SAI6_ROOT 201 +#define IMX8MQ_CLK_UART1_ROOT 202 +#define IMX8MQ_CLK_UART2_ROOT 203 +#define IMX8MQ_CLK_UART3_ROOT 204 +#define IMX8MQ_CLK_UART4_ROOT 205 +#define IMX8MQ_CLK_USB1_CTRL_ROOT 206 +#define IMX8MQ_CLK_USB2_CTRL_ROOT 207 +#define IMX8MQ_CLK_USB1_PHY_ROOT 208 +#define IMX8MQ_CLK_USB2_PHY_ROOT 209 +#define IMX8MQ_CLK_USDHC1_ROOT 210 +#define IMX8MQ_CLK_USDHC2_ROOT 211 +#define IMX8MQ_CLK_WDOG1_ROOT 212 +#define IMX8MQ_CLK_WDOG2_ROOT 213 +#define IMX8MQ_CLK_WDOG3_ROOT 214 +#define IMX8MQ_CLK_GPU_ROOT 215 +#define IMX8MQ_CLK_HEVC_ROOT 216 +#define IMX8MQ_CLK_AVC_ROOT 217 +#define IMX8MQ_CLK_VP9_ROOT 218 +#define IMX8MQ_CLK_HEVC_INTER_ROOT 219 +#define IMX8MQ_CLK_DISP_ROOT 220 +#define IMX8MQ_CLK_HDMI_ROOT 221 +#define IMX8MQ_CLK_HDMI_PHY_ROOT 222 +#define IMX8MQ_CLK_VPU_DEC_ROOT 223 +#define IMX8MQ_CLK_CSI1_ROOT 224 +#define IMX8MQ_CLK_CSI2_ROOT 225 +#define IMX8MQ_CLK_RAWNAND_ROOT 226 +#define IMX8MQ_CLK_SDMA1_ROOT 227 +#define IMX8MQ_CLK_SDMA2_ROOT 228 +#define IMX8MQ_CLK_VPU_G1_ROOT 229 +#define IMX8MQ_CLK_VPU_G2_ROOT 230 /* SCCG PLL GATE */ -#define IMX8MQ_SYS1_PLL_OUT 242 -#define IMX8MQ_SYS2_PLL_OUT 243 -#define IMX8MQ_SYS3_PLL_OUT 244 -#define IMX8MQ_DRAM_PLL_OUT 245 - -#define IMX8MQ_GPT_3M_CLK 246 - -#define IMX8MQ_CLK_IPG_ROOT 247 -#define IMX8MQ_CLK_IPG_AUDIO_ROOT 248 -#define IMX8MQ_CLK_SAI1_IPG 249 -#define IMX8MQ_CLK_SAI2_IPG 250 -#define IMX8MQ_CLK_SAI3_IPG 251 -#define IMX8MQ_CLK_SAI4_IPG 252 -#define IMX8MQ_CLK_SAI5_IPG 253 -#define IMX8MQ_CLK_SAI6_IPG 254 +#define IMX8MQ_SYS1_PLL_OUT 231 +#define IMX8MQ_SYS2_PLL_OUT 232 +#define IMX8MQ_SYS3_PLL_OUT 233 +#define IMX8MQ_DRAM_PLL_OUT 234 + +#define IMX8MQ_GPT_3M_CLK 235 + +#define IMX8MQ_CLK_IPG_ROOT 236 +#define IMX8MQ_CLK_IPG_AUDIO_ROOT 237 +#define IMX8MQ_CLK_SAI1_IPG 238 +#define IMX8MQ_CLK_SAI2_IPG 239 +#define IMX8MQ_CLK_SAI3_IPG 240 +#define IMX8MQ_CLK_SAI4_IPG 241 +#define IMX8MQ_CLK_SAI5_IPG 242 +#define IMX8MQ_CLK_SAI6_IPG 243 /* DSI AHB/IPG clocks */ /* rxesc clock */ -#define IMX8MQ_CLK_DSI_AHB 255 +#define IMX8MQ_CLK_DSI_AHB 244 /* txesc clock */ -#define IMX8MQ_CLK_DSI_IPG_DIV 256 +#define IMX8MQ_CLK_DSI_IPG_DIV 245 -#define IMX8MQ_CLK_TMU_ROOT 257 +#define IMX8MQ_CLK_TMU_ROOT 246 /* Display root clocks */ -#define IMX8MQ_CLK_DISP_AXI_ROOT 258 -#define IMX8MQ_CLK_DISP_APB_ROOT 259 -#define IMX8MQ_CLK_DISP_RTRM_ROOT 260 +#define IMX8MQ_CLK_DISP_AXI_ROOT 247 +#define IMX8MQ_CLK_DISP_APB_ROOT 248 +#define IMX8MQ_CLK_DISP_RTRM_ROOT 249 -#define IMX8MQ_CLK_OCOTP_ROOT 261 +#define IMX8MQ_CLK_OCOTP_ROOT 250 -#define IMX8MQ_CLK_DRAM_ALT_ROOT 262 -#define IMX8MQ_CLK_DRAM_CORE 263 +#define IMX8MQ_CLK_DRAM_ALT_ROOT 251 +#define IMX8MQ_CLK_DRAM_CORE 252 -#define IMX8MQ_CLK_MU_ROOT 264 -#define IMX8MQ_VIDEO2_PLL_OUT 265 +#define IMX8MQ_CLK_MU_ROOT 253 +#define IMX8MQ_VIDEO2_PLL_OUT 254 -#define IMX8MQ_CLK_CLKO2 266 +#define IMX8MQ_CLK_CLKO2 255 -#define IMX8MQ_CLK_NAND_USDHC_BUS_RAWNAND_CLK 267 +#define IMX8MQ_CLK_NAND_USDHC_BUS_RAWNAND_CLK 256 -#define IMX8MQ_CLK_END 268 +#define IMX8MQ_CLK_CLKO1 257 +#define IMX8MQ_CLK_ARM 258 + +#define IMX8MQ_CLK_GPIO1_ROOT 259 +#define IMX8MQ_CLK_GPIO2_ROOT 260 +#define IMX8MQ_CLK_GPIO3_ROOT 261 +#define IMX8MQ_CLK_GPIO4_ROOT 262 +#define IMX8MQ_CLK_GPIO5_ROOT 263 + +#define IMX8MQ_CLK_END 264 #endif /* __DT_BINDINGS_CLOCK_IMX8MQ_H */ diff --git a/include/dt-bindings/clock/marvell,mmp2.h b/include/dt-bindings/clock/marvell,mmp2.h index 228a5e234af04..e785c6eb35613 100644 --- a/include/dt-bindings/clock/marvell,mmp2.h +++ b/include/dt-bindings/clock/marvell,mmp2.h @@ -71,6 +71,7 @@ #define MMP2_CLK_CCIC1_MIX 117 #define MMP2_CLK_CCIC1_PHY 118 #define MMP2_CLK_CCIC1_SPHY 119 +#define MMP2_CLK_DISP0_LCDC 120 #define MMP2_NR_CLKS 200 #endif diff --git a/include/dt-bindings/clock/meson8b-clkc.h b/include/dt-bindings/clock/meson8b-clkc.h index 5fe2923382d01..8067077a62ca9 100644 --- a/include/dt-bindings/clock/meson8b-clkc.h +++ b/include/dt-bindings/clock/meson8b-clkc.h @@ -104,6 +104,7 @@ #define CLKID_MPLL2 95 #define CLKID_NAND_CLK 112 #define CLKID_ABP 124 +#define CLKID_APB 124 #define CLKID_PERIPH 126 #define CLKID_AXI 128 #define CLKID_L2_DRAM 130 diff --git a/include/dt-bindings/clock/mt2712-clk.h b/include/dt-bindings/clock/mt2712-clk.h index 76265836a1e1d..c3b29dff9c0ee 100644 --- a/include/dt-bindings/clock/mt2712-clk.h +++ b/include/dt-bindings/clock/mt2712-clk.h @@ -228,7 +228,8 @@ #define CLK_TOP_NFI2X_EN 189 #define CLK_TOP_NFIECC_EN 190 #define CLK_TOP_NFI1X_CK_EN 191 -#define CLK_TOP_NR_CLK 192 +#define CLK_TOP_APLL2_D3 192 +#define CLK_TOP_NR_CLK 193 /* INFRACFG */ diff --git a/include/dt-bindings/clock/mt8173-clk.h b/include/dt-bindings/clock/mt8173-clk.h index 8aea623dd518e..76e4e5b65353d 100644 --- a/include/dt-bindings/clock/mt8173-clk.h +++ b/include/dt-bindings/clock/mt8173-clk.h @@ -194,7 +194,8 @@ #define CLK_INFRA_PMICWRAP 11 #define CLK_INFRA_CLK_13M 12 #define CLK_INFRA_CA53SEL 13 -#define CLK_INFRA_CA57SEL 14 +#define CLK_INFRA_CA57SEL 14 /* Deprecated. Don't use it. */ +#define CLK_INFRA_CA72SEL 14 #define CLK_INFRA_NR_CLK 15 /* PERI_SYS */ diff --git a/include/dt-bindings/clock/qcom,rpmcc.h b/include/dt-bindings/clock/qcom,rpmcc.h index 3658b0c149667..ede93a0ca1560 100644 --- a/include/dt-bindings/clock/qcom,rpmcc.h +++ b/include/dt-bindings/clock/qcom,rpmcc.h @@ -127,5 +127,15 @@ #define RPM_SMD_BIMC_GPU_A_CLK 77 #define RPM_SMD_QPIC_CLK 78 #define RPM_SMD_QPIC_CLK_A 79 +#define RPM_SMD_LN_BB_CLK1 80 +#define RPM_SMD_LN_BB_CLK1_A 81 +#define RPM_SMD_LN_BB_CLK2 82 +#define RPM_SMD_LN_BB_CLK2_A 83 +#define RPM_SMD_LN_BB_CLK3_PIN 84 +#define RPM_SMD_LN_BB_CLK3_A_PIN 85 +#define RPM_SMD_RF_CLK3 86 +#define RPM_SMD_RF_CLK3_A 87 +#define RPM_SMD_RF_CLK3_PIN 88 +#define RPM_SMD_RF_CLK3_A_PIN 89 #endif diff --git a/include/dt-bindings/clock/qcom,rpmh.h b/include/dt-bindings/clock/qcom,rpmh.h index f48fbd6f20952..edcab3f7b7d33 100644 --- a/include/dt-bindings/clock/qcom,rpmh.h +++ b/include/dt-bindings/clock/qcom,rpmh.h @@ -18,5 +18,6 @@ #define RPMH_RF_CLK2_A 9 #define RPMH_RF_CLK3 10 #define RPMH_RF_CLK3_A 11 +#define RPMH_IPA_CLK 12 #endif diff --git a/include/dt-bindings/clock/r8a774a1-cpg-mssr.h b/include/dt-bindings/clock/r8a774a1-cpg-mssr.h index 9bc5d45ff4b59..e355363f40c2f 100644 --- a/include/dt-bindings/clock/r8a774a1-cpg-mssr.h +++ b/include/dt-bindings/clock/r8a774a1-cpg-mssr.h @@ -54,5 +54,6 @@ #define R8A774A1_CLK_CPEX 43 #define R8A774A1_CLK_R 44 #define R8A774A1_CLK_OSC 45 +#define R8A774A1_CLK_CANFD 46 #endif /* __DT_BINDINGS_CLOCK_R8A774A1_CPG_MSSR_H__ */ diff --git a/include/dt-bindings/clock/r8a774c0-cpg-mssr.h b/include/dt-bindings/clock/r8a774c0-cpg-mssr.h index 8fe51b6aca284..8ad9cd6be8e9c 100644 --- a/include/dt-bindings/clock/r8a774c0-cpg-mssr.h +++ b/include/dt-bindings/clock/r8a774c0-cpg-mssr.h @@ -56,5 +56,6 @@ #define R8A774C0_CLK_CSI0 45 #define R8A774C0_CLK_CP 46 #define R8A774C0_CLK_CPEX 47 +#define R8A774C0_CLK_CANFD 48 #endif /* __DT_BINDINGS_CLOCK_R8A774C0_CPG_MSSR_H__ */ diff --git a/include/dt-bindings/clock/stm32mp1-clks.h b/include/dt-bindings/clock/stm32mp1-clks.h index 90ec780bfc686..4cdaf135829c6 100644 --- a/include/dt-bindings/clock/stm32mp1-clks.h +++ b/include/dt-bindings/clock/stm32mp1-clks.h @@ -248,7 +248,4 @@ #define STM32MP1_LAST_CLK 232 -#define LTDC_K LTDC_PX -#define ETHMAC_K ETHCK_K - #endif /* _DT_BINDINGS_STM32MP1_CLKS_H_ */ diff --git a/include/dt-bindings/reset/g12a-aoclkc.h b/include/dt-bindings/reset/g12a-aoclkc.h new file mode 100644 index 0000000000000..bd2e2337135cd --- /dev/null +++ b/include/dt-bindings/reset/g12a-aoclkc.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ +/* + * Copyright (c) 2016 BayLibre, SAS + * Author: Neil Armstrong <narmstrong@baylibre.com> + */ + +#ifndef DT_BINDINGS_RESET_AMLOGIC_MESON_G12A_AOCLK +#define DT_BINDINGS_RESET_AMLOGIC_MESON_G12A_AOCLK + +#define RESET_AO_IR_IN 0 +#define RESET_AO_UART 1 +#define RESET_AO_I2C_M 2 +#define RESET_AO_I2C_S 3 +#define RESET_AO_SAR_ADC 4 +#define RESET_AO_UART2 5 +#define RESET_AO_IR_OUT 6 + +#endif diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h index 33771352dcd69..05a18dd265b55 100644 --- a/include/kvm/arm_arch_timer.h +++ b/include/kvm/arm_arch_timer.h @@ -22,7 +22,22 @@ #include <linux/clocksource.h> #include <linux/hrtimer.h> +enum kvm_arch_timers { + TIMER_PTIMER, + TIMER_VTIMER, + NR_KVM_TIMERS +}; + +enum kvm_arch_timer_regs { + TIMER_REG_CNT, + TIMER_REG_CVAL, + TIMER_REG_TVAL, + TIMER_REG_CTL, +}; + struct arch_timer_context { + struct kvm_vcpu *vcpu; + /* Registers: control register, timer value */ u32 cnt_ctl; u64 cnt_cval; @@ -30,30 +45,36 @@ struct arch_timer_context { /* Timer IRQ */ struct kvm_irq_level irq; + /* Virtual offset */ + u64 cntvoff; + + /* Emulated Timer (may be unused) */ + struct hrtimer hrtimer; + /* - * We have multiple paths which can save/restore the timer state - * onto the hardware, so we need some way of keeping track of - * where the latest state is. - * - * loaded == true: State is loaded on the hardware registers. - * loaded == false: State is stored in memory. + * We have multiple paths which can save/restore the timer state onto + * the hardware, so we need some way of keeping track of where the + * latest state is. */ - bool loaded; + bool loaded; - /* Virtual offset */ - u64 cntvoff; + /* Duplicated state from arch_timer.c for convenience */ + u32 host_timer_irq; + u32 host_timer_irq_flags; +}; + +struct timer_map { + struct arch_timer_context *direct_vtimer; + struct arch_timer_context *direct_ptimer; + struct arch_timer_context *emul_ptimer; }; struct arch_timer_cpu { - struct arch_timer_context vtimer; - struct arch_timer_context ptimer; + struct arch_timer_context timers[NR_KVM_TIMERS]; /* Background timer used when the guest is not running */ struct hrtimer bg_timer; - /* Physical timer emulation */ - struct hrtimer phys_timer; - /* Is the timer enabled */ bool enabled; }; @@ -76,9 +97,6 @@ int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); bool kvm_timer_is_pending(struct kvm_vcpu *vcpu); -void kvm_timer_schedule(struct kvm_vcpu *vcpu); -void kvm_timer_unschedule(struct kvm_vcpu *vcpu); - u64 kvm_phys_timer_read(void); void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu); @@ -88,7 +106,19 @@ void kvm_timer_init_vhe(void); bool kvm_arch_timer_get_input_level(int vintid); -#define vcpu_vtimer(v) (&(v)->arch.timer_cpu.vtimer) -#define vcpu_ptimer(v) (&(v)->arch.timer_cpu.ptimer) +#define vcpu_timer(v) (&(v)->arch.timer_cpu) +#define vcpu_get_timer(v,t) (&vcpu_timer(v)->timers[(t)]) +#define vcpu_vtimer(v) (&(v)->arch.timer_cpu.timers[TIMER_VTIMER]) +#define vcpu_ptimer(v) (&(v)->arch.timer_cpu.timers[TIMER_PTIMER]) + +#define arch_timer_ctx_index(ctx) ((ctx) - vcpu_timer((ctx)->vcpu)->timers) + +u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu, + enum kvm_arch_timers tmr, + enum kvm_arch_timer_regs treg); +void kvm_arm_timer_write_sysreg(struct kvm_vcpu *vcpu, + enum kvm_arch_timers tmr, + enum kvm_arch_timer_regs treg, + u64 val); #endif diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 6ac47f5ea514a..d5dcebd7aad33 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -400,12 +400,17 @@ extern bool acpi_osi_is_win8(void); #ifdef CONFIG_ACPI_NUMA int acpi_map_pxm_to_online_node(int pxm); +int acpi_map_pxm_to_node(int pxm); int acpi_get_node(acpi_handle handle); #else static inline int acpi_map_pxm_to_online_node(int pxm) { return 0; } +static inline int acpi_map_pxm_to_node(int pxm) +{ + return 0; +} static inline int acpi_get_node(acpi_handle handle) { return 0; diff --git a/include/linux/amba/bus.h b/include/linux/amba/bus.h index d143c13bed26d..f99b74a6e4cab 100644 --- a/include/linux/amba/bus.h +++ b/include/linux/amba/bus.h @@ -25,6 +25,43 @@ #define AMBA_CID 0xb105f00d #define CORESIGHT_CID 0xb105900d +/* + * CoreSight Architecture specification updates the ID specification + * for components on the AMBA bus. (ARM IHI 0029E) + * + * Bits 15:12 of the CID are the device class. + * + * Class 0xF remains for PrimeCell and legacy components. (AMBA_CID above) + * Class 0x9 defines the component as CoreSight (CORESIGHT_CID above) + * Class 0x0, 0x1, 0xB, 0xE define components that do not have driver support + * at present. + * Class 0x2-0x8,0xA and 0xD-0xD are presently reserved. + * + * Remaining CID bits stay as 0xb105-00d + */ + +/** + * Class 0x9 components use additional values to form a Unique Component + * Identifier (UCI), where peripheral ID values are identical for different + * components. Passed to the amba bus code from the component driver via + * the amba_id->data pointer. + * @devarch : coresight devarch register value + * @devarch_mask: mask bits used for matching. 0 indicates UCI not used. + * @devtype : coresight device type value + * @data : additional driver data. As we have usurped the original + * pointer some devices may still need additional data + */ +struct amba_cs_uci_id { + unsigned int devarch; + unsigned int devarch_mask; + unsigned int devtype; + void *data; +}; + +/* define offsets for registers used by UCI */ +#define UCI_REG_DEVTYPE_OFFSET 0xFCC +#define UCI_REG_DEVARCH_OFFSET 0xFBC + struct clk; struct amba_device { @@ -32,6 +69,8 @@ struct amba_device { struct resource res; struct clk *pclk; unsigned int periphid; + unsigned int cid; + struct amba_cs_uci_id uci; unsigned int irq[AMBA_NR_IRQS]; char *driver_override; }; diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index e443fa9fa8590..b7cf80a712939 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -792,6 +792,9 @@ unsigned int __clk_get_enable_count(struct clk *clk); unsigned long clk_hw_get_rate(const struct clk_hw *hw); unsigned long __clk_get_flags(struct clk *clk); unsigned long clk_hw_get_flags(const struct clk_hw *hw); +#define clk_hw_can_set_rate_parent(hw) \ + (clk_hw_get_flags((hw)) & CLK_SET_RATE_PARENT) + bool clk_hw_is_prepared(const struct clk_hw *hw); bool clk_hw_rate_is_protected(const struct clk_hw *hw); bool clk_hw_is_enabled(const struct clk_hw *hw); diff --git a/include/linux/clk.h b/include/linux/clk.h index a7773b5c0b9fb..d8bc1a856b39c 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -384,6 +384,17 @@ int __must_check devm_clk_bulk_get_all(struct device *dev, struct clk *devm_clk_get(struct device *dev, const char *id); /** + * devm_clk_get_optional - lookup and obtain a managed reference to an optional + * clock producer. + * @dev: device for clock "consumer" + * @id: clock consumer ID + * + * Behaves the same as devm_clk_get() except where there is no clock producer. + * In this case, instead of returning -ENOENT, the function returns NULL. + */ +struct clk *devm_clk_get_optional(struct device *dev, const char *id); + +/** * devm_get_clk_from_child - lookup and obtain a managed reference to a * clock producer from child node. * @dev: device for clock "consumer" @@ -718,6 +729,12 @@ static inline struct clk *devm_clk_get(struct device *dev, const char *id) return NULL; } +static inline struct clk *devm_clk_get_optional(struct device *dev, + const char *id) +{ + return NULL; +} + static inline int __must_check devm_clk_bulk_get(struct device *dev, int num_clks, struct clk_bulk_data *clks) { @@ -862,6 +879,25 @@ static inline void clk_bulk_disable_unprepare(int num_clks, clk_bulk_unprepare(num_clks, clks); } +/** + * clk_get_optional - lookup and obtain a reference to an optional clock + * producer. + * @dev: device for clock "consumer" + * @id: clock consumer ID + * + * Behaves the same as clk_get() except where there is no clock producer. In + * this case, instead of returning -ENOENT, the function returns NULL. + */ +static inline struct clk *clk_get_optional(struct device *dev, const char *id) +{ + struct clk *clk = clk_get(dev, id); + + if (clk == ERR_PTR(-ENOENT)) + return NULL; + + return clk; +} + #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK) struct clk *of_clk_get(struct device_node *np, int index); struct clk *of_clk_get_by_name(struct device_node *np, const char *name); diff --git a/include/linux/clk/ti.h b/include/linux/clk/ti.h index eacc5df57b99f..78872efc7be09 100644 --- a/include/linux/clk/ti.h +++ b/include/linux/clk/ti.h @@ -160,6 +160,7 @@ struct clk_hw_omap { struct clockdomain *clkdm; const struct clk_hw_omap_ops *ops; u32 context; + int autoidle_count; }; /* diff --git a/include/linux/clkdev.h b/include/linux/clkdev.h index 4890ff0332202..ccb32af5848b8 100644 --- a/include/linux/clkdev.h +++ b/include/linux/clkdev.h @@ -52,4 +52,8 @@ int clk_add_alias(const char *, const char *, const char *, struct device *); int clk_register_clkdev(struct clk *, const char *, const char *); int clk_hw_register_clkdev(struct clk_hw *, const char *, const char *); +int devm_clk_hw_register_clkdev(struct device *dev, struct clk_hw *hw, + const char *con_id, const char *dev_id); +void devm_clk_release_clkdev(struct device *dev, const char *con_id, + const char *dev_id); #endif diff --git a/include/linux/dma/dw.h b/include/linux/dma/dw.h index e166cac8e870e..9752f3745f764 100644 --- a/include/linux/dma/dw.h +++ b/include/linux/dma/dw.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Driver for the Synopsys DesignWare DMA Controller * * Copyright (C) 2007 Atmel Corporation * Copyright (C) 2010-2011 ST Microelectronics * Copyright (C) 2014 Intel Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef _DMA_DW_H #define _DMA_DW_H @@ -45,9 +42,13 @@ struct dw_dma_chip { #if IS_ENABLED(CONFIG_DW_DMAC_CORE) int dw_dma_probe(struct dw_dma_chip *chip); int dw_dma_remove(struct dw_dma_chip *chip); +int idma32_dma_probe(struct dw_dma_chip *chip); +int idma32_dma_remove(struct dw_dma_chip *chip); #else static inline int dw_dma_probe(struct dw_dma_chip *chip) { return -ENODEV; } static inline int dw_dma_remove(struct dw_dma_chip *chip) { return 0; } +static inline int idma32_dma_probe(struct dw_dma_chip *chip) { return -ENODEV; } +static inline int idma32_dma_remove(struct dw_dma_chip *chip) { return 0; } #endif /* CONFIG_DW_DMAC_CORE */ #endif /* _DMA_DW_H */ diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index d7711048ef93b..f5740423b0023 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -116,6 +116,7 @@ struct f2fs_super_block { /* * For checkpoint */ +#define CP_DISABLED_QUICK_FLAG 0x00002000 #define CP_DISABLED_FLAG 0x00001000 #define CP_QUOTA_NEED_FSCK_FLAG 0x00000800 #define CP_LARGE_NAT_BITMAP_FLAG 0x00000400 @@ -186,7 +187,7 @@ struct f2fs_orphan_block { struct f2fs_extent { __le32 fofs; /* start file offset of the extent */ __le32 blk; /* start block address of the extent */ - __le32 len; /* lengh of the extent */ + __le32 len; /* length of the extent */ } __packed; #define F2FS_NAME_LEN 255 @@ -284,7 +285,7 @@ enum { struct node_footer { __le32 nid; /* node id */ - __le32 ino; /* inode nunmber */ + __le32 ino; /* inode number */ __le32 flag; /* include cold/fsync/dentry marks and offset */ __le64 cp_ver; /* checkpoint version */ __le32 next_blkaddr; /* next node page block address */ @@ -489,12 +490,12 @@ typedef __le32 f2fs_hash_t; /* * space utilization of regular dentry and inline dentry (w/o extra reservation) - * regular dentry inline dentry - * bitmap 1 * 27 = 27 1 * 23 = 23 - * reserved 1 * 3 = 3 1 * 7 = 7 - * dentry 11 * 214 = 2354 11 * 182 = 2002 - * filename 8 * 214 = 1712 8 * 182 = 1456 - * total 4096 3488 + * regular dentry inline dentry (def) inline dentry (min) + * bitmap 1 * 27 = 27 1 * 23 = 23 1 * 1 = 1 + * reserved 1 * 3 = 3 1 * 7 = 7 1 * 1 = 1 + * dentry 11 * 214 = 2354 11 * 182 = 2002 11 * 2 = 22 + * filename 8 * 214 = 1712 8 * 182 = 1456 8 * 2 = 16 + * total 4096 3488 40 * * Note: there are more reserved space in inline dentry than in regular * dentry, when converting inline dentry we should handle this carefully. @@ -506,12 +507,13 @@ typedef __le32 f2fs_hash_t; #define SIZE_OF_RESERVED (PAGE_SIZE - ((SIZE_OF_DIR_ENTRY + \ F2FS_SLOT_LEN) * \ NR_DENTRY_IN_BLOCK + SIZE_OF_DENTRY_BITMAP)) +#define MIN_INLINE_DENTRY_SIZE 40 /* just include '.' and '..' entries */ /* One directory entry slot representing F2FS_SLOT_LEN-sized file name */ struct f2fs_dir_entry { __le32 hash_code; /* hash code of file name */ __le32 ino; /* inode number */ - __le16 name_len; /* lengh of file name */ + __le16 name_len; /* length of file name */ __u8 file_type; /* file type */ } __packed; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index c38cc5eb7e737..9d55c63db09b5 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -48,6 +48,27 @@ */ #define KVM_MEMSLOT_INVALID (1UL << 16) +/* + * Bit 63 of the memslot generation number is an "update in-progress flag", + * e.g. is temporarily set for the duration of install_new_memslots(). + * This flag effectively creates a unique generation number that is used to + * mark cached memslot data, e.g. MMIO accesses, as potentially being stale, + * i.e. may (or may not) have come from the previous memslots generation. + * + * This is necessary because the actual memslots update is not atomic with + * respect to the generation number update. Updating the generation number + * first would allow a vCPU to cache a spte from the old memslots using the + * new generation number, and updating the generation number after switching + * to the new memslots would allow cache hits using the old generation number + * to reference the defunct memslots. + * + * This mechanism is used to prevent getting hits in KVM's caches while a + * memslot update is in-progress, and to prevent cache hits *after* updating + * the actual generation number against accesses that were inserted into the + * cache *before* the memslots were updated. + */ +#define KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS BIT_ULL(63) + /* Two fragments for cross MMIO pages. */ #define KVM_MAX_MMIO_FRAGMENTS 2 @@ -634,7 +655,7 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, struct kvm_memory_slot *dont); int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, unsigned long npages); -void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots); +void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen); int kvm_arch_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *memslot, const struct kvm_userspace_memory_region *mem, @@ -1182,6 +1203,7 @@ extern bool kvm_rebooting; extern unsigned int halt_poll_ns; extern unsigned int halt_poll_ns_grow; +extern unsigned int halt_poll_ns_grow_start; extern unsigned int halt_poll_ns_shrink; struct kvm_device { diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 43348303cb4bf..feb342d026f2e 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -130,6 +130,7 @@ struct nd_region_desc { void *provider_data; int num_lanes; int numa_node; + int target_node; unsigned long flags; struct device_node *of_node; }; diff --git a/include/linux/ntb.h b/include/linux/ntb.h index 181d16601dd9c..56a92e3ae3ae5 100644 --- a/include/linux/ntb.h +++ b/include/linux/ntb.h @@ -296,7 +296,8 @@ struct ntb_dev_ops { int (*db_clear_mask)(struct ntb_dev *ntb, u64 db_bits); int (*peer_db_addr)(struct ntb_dev *ntb, - phys_addr_t *db_addr, resource_size_t *db_size); + phys_addr_t *db_addr, resource_size_t *db_size, + u64 *db_data, int db_bit); u64 (*peer_db_read)(struct ntb_dev *ntb); int (*peer_db_set)(struct ntb_dev *ntb, u64 db_bits); int (*peer_db_clear)(struct ntb_dev *ntb, u64 db_bits); @@ -1078,6 +1079,8 @@ static inline int ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) * @ntb: NTB device context. * @db_addr: OUT - The address of the peer doorbell register. * @db_size: OUT - The number of bytes to write the peer doorbell register. + * @db_data: OUT - The data of peer doorbell register + * @db_bit: door bell bit number * * Return the address of the peer doorbell register. This may be used, for * example, by drivers that offload memory copy operations to a dma engine. @@ -1091,12 +1094,13 @@ static inline int ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) */ static inline int ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, - resource_size_t *db_size) + resource_size_t *db_size, + u64 *db_data, int db_bit) { if (!ntb->ops->peer_db_addr) return -EINVAL; - return ntb->ops->peer_db_addr(ntb, db_addr, db_size); + return ntb->ops->peer_db_addr(ntb, db_addr, db_size, db_data, db_bit); } /** diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index b477a70cc2e45..bcf909d0de5f8 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -239,6 +239,7 @@ pgoff_t page_cache_prev_miss(struct address_space *mapping, #define FGP_WRITE 0x00000008 #define FGP_NOFS 0x00000010 #define FGP_NOWAIT 0x00000020 +#define FGP_FOR_MMAP 0x00000040 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset, int fgp_flags, gfp_t cache_gfp_mask); diff --git a/include/linux/platform_data/dma-dw.h b/include/linux/platform_data/dma-dw.h index 1a1d58ebffbf1..f3eaf9ec00a1b 100644 --- a/include/linux/platform_data/dma-dw.h +++ b/include/linux/platform_data/dma-dw.h @@ -1,12 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Driver for the Synopsys DesignWare DMA Controller * * Copyright (C) 2007 Atmel Corporation * Copyright (C) 2010-2011 ST Microelectronics - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef _PLATFORM_DATA_DMA_DW_H #define _PLATFORM_DATA_DMA_DW_H @@ -38,10 +35,6 @@ struct dw_dma_slave { /** * struct dw_dma_platform_data - Controller configuration parameters * @nr_channels: Number of channels supported by hardware (max 8) - * @is_private: The device channels should be marked as private and not for - * by the general purpose DMA channel allocator. - * @is_memcpy: The device channels do support memory-to-memory transfers. - * @is_idma32: The type of the DMA controller is iDMA32 * @chan_allocation_order: Allocate channels starting from 0 or 7 * @chan_priority: Set channel priority increasing from 0 to 7 or 7 to 0. * @block_size: Maximum block size supported by the controller @@ -53,9 +46,6 @@ struct dw_dma_slave { */ struct dw_dma_platform_data { unsigned int nr_channels; - bool is_private; - bool is_memcpy; - bool is_idma32; #define CHAN_ALLOCATION_ASCENDING 0 /* zero to seven */ #define CHAN_ALLOCATION_DESCENDING 1 /* seven to zero */ unsigned char chan_allocation_order; diff --git a/include/linux/platform_data/dma-imx.h b/include/linux/platform_data/dma-imx.h index 7d964e787299e..9daea8d42a10b 100644 --- a/include/linux/platform_data/dma-imx.h +++ b/include/linux/platform_data/dma-imx.h @@ -55,6 +55,7 @@ struct imx_dma_data { int dma_request2; /* secondary DMA request line */ enum sdma_peripheral_type peripheral_type; int priority; + struct device_node *of_node; }; static inline int imx_dma_is_ipu(struct dma_chan *chan) diff --git a/include/linux/platform_data/clk-lpss.h b/include/linux/platform_data/x86/clk-lpss.h index 23901992b9ddc..23901992b9ddc 100644 --- a/include/linux/platform_data/clk-lpss.h +++ b/include/linux/platform_data/x86/clk-lpss.h diff --git a/include/linux/pm.h b/include/linux/pm.h index 06f7ed8939286..66c19a65a5140 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -643,7 +643,6 @@ struct dev_pm_info { struct dev_pm_qos *qos; }; -extern void update_pm_runtime_accounting(struct device *dev); extern int dev_pm_get_subsys_data(struct device *dev); extern void dev_pm_put_subsys_data(struct device *dev); diff --git a/include/linux/pm_wakeup.h b/include/linux/pm_wakeup.h index 4238dde0aaf03..0ff134d6575a9 100644 --- a/include/linux/pm_wakeup.h +++ b/include/linux/pm_wakeup.h @@ -96,7 +96,6 @@ static inline void device_set_wakeup_path(struct device *dev) /* drivers/base/power/wakeup.c */ extern void wakeup_source_prepare(struct wakeup_source *ws, const char *name); extern struct wakeup_source *wakeup_source_create(const char *name); -extern void wakeup_source_drop(struct wakeup_source *ws); extern void wakeup_source_destroy(struct wakeup_source *ws); extern void wakeup_source_add(struct wakeup_source *ws); extern void wakeup_source_remove(struct wakeup_source *ws); @@ -134,8 +133,6 @@ static inline struct wakeup_source *wakeup_source_create(const char *name) return NULL; } -static inline void wakeup_source_drop(struct wakeup_source *ws) {} - static inline void wakeup_source_destroy(struct wakeup_source *ws) {} static inline void wakeup_source_add(struct wakeup_source *ws) {} @@ -204,12 +201,6 @@ static inline void wakeup_source_init(struct wakeup_source *ws, wakeup_source_add(ws); } -static inline void wakeup_source_trash(struct wakeup_source *ws) -{ - wakeup_source_remove(ws); - wakeup_source_drop(ws); -} - static inline void __pm_wakeup_event(struct wakeup_source *ws, unsigned int msec) { return pm_wakeup_ws_event(ws, msec, false); diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index d0e1f1522a78e..52a283ba0465a 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -73,6 +73,7 @@ struct proc_dir_entry *proc_create_net_single_write(const char *name, umode_t mo int (*show)(struct seq_file *, void *), proc_write_t write, void *data); +extern struct pid *tgid_pidfd_to_pid(const struct file *file); #else /* CONFIG_PROC_FS */ @@ -114,6 +115,11 @@ static inline int remove_proc_subtree(const char *name, struct proc_dir_entry *p #define proc_create_net(name, mode, parent, state_size, ops) ({NULL;}) #define proc_create_net_single(name, mode, parent, show, data) ({NULL;}) +static inline struct pid *tgid_pidfd_to_pid(const struct file *file) +{ + return ERR_PTR(-EBADF); +} + #endif /* CONFIG_PROC_FS */ struct net; diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index 507a2b5242085..04d04709f2bda 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h @@ -345,9 +345,9 @@ struct firmware; * @stop: power off the device * @kick: kick a virtqueue (virtqueue id given as a parameter) * @da_to_va: optional platform hook to perform address translations - * @load_rsc_table: load resource table from firmware image + * @parse_fw: parse firmware to extract information (e.g. resource table) * @find_loaded_rsc_table: find the loaded resouce table - * @load: load firmeware to memory, where the remote processor + * @load: load firmware to memory, where the remote processor * expects to find it * @sanity_check: sanity check the fw image * @get_boot_addr: get boot address to entry point specified in firmware @@ -554,11 +554,11 @@ struct rproc_vdev { struct kref refcount; struct rproc_subdev subdev; + struct device dev; unsigned int id; struct list_head node; struct rproc *rproc; - struct virtio_device vdev; struct rproc_vring vring[RVDEV_NUM_VRINGS]; u32 rsc_offset; u32 index; @@ -601,7 +601,7 @@ int rproc_coredump_add_custom_segment(struct rproc *rproc, static inline struct rproc_vdev *vdev_to_rvdev(struct virtio_device *vdev) { - return container_of(vdev, struct rproc_vdev, vdev); + return container_of(vdev->dev.parent, struct rproc_vdev, dev); } static inline struct rproc *vdev_to_rproc(struct virtio_device *vdev) diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index f1429675f2528..1a40277b512c9 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -128,7 +128,7 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts, unsigned long *lost_events); struct ring_buffer_iter * -ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu); +ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu, gfp_t flags); void ring_buffer_read_prepare_sync(void); void ring_buffer_read_start(struct ring_buffer_iter *iter); void ring_buffer_read_finish(struct ring_buffer_iter *iter); diff --git a/include/linux/swap.h b/include/linux/swap.h index fc50e21b3b88e..4bfb5c4ac108d 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -157,9 +157,9 @@ struct swap_extent { /* * Max bad pages in the new format.. */ -#define __swapoffset(x) ((unsigned long)&((union swap_header *)0)->x) #define MAX_SWAP_BADPAGES \ - ((__swapoffset(magic.magic) - __swapoffset(info.badpages)) / sizeof(int)) + ((offsetof(union swap_header, magic.magic) - \ + offsetof(union swap_header, info.badpages)) / sizeof(int)) enum { SWP_USED = (1 << 0), /* is slot in swap_info[] used? */ diff --git a/include/linux/switchtec.h b/include/linux/switchtec.h index eee0412bdf4b8..52a079b3a9a6d 100644 --- a/include/linux/switchtec.h +++ b/include/linux/switchtec.h @@ -248,9 +248,13 @@ struct ntb_ctrl_regs { u32 win_size; u64 xlate_addr; } bar_entry[6]; - u32 reserved2[216]; - u32 req_id_table[256]; - u32 reserved3[512]; + struct { + u32 win_size; + u32 reserved[3]; + } bar_ext_entry[6]; + u32 reserved2[192]; + u32 req_id_table[512]; + u32 reserved3[256]; u64 lut_entry[512]; } __packed; diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index c2962953bf11c..e446806a561f8 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -985,6 +985,9 @@ asmlinkage long sys_statx(int dfd, const char __user *path, unsigned flags, unsigned mask, struct statx __user *buffer); asmlinkage long sys_rseq(struct rseq __user *rseq, uint32_t rseq_len, int flags, uint32_t sig); +asmlinkage long sys_pidfd_send_signal(int pidfd, int sig, + siginfo_t __user *info, + unsigned int flags); /* * Architecture-specific system calls diff --git a/include/linux/vgaarb.h b/include/linux/vgaarb.h index ee162e3e879b9..553b34c8b5f70 100644 --- a/include/linux/vgaarb.h +++ b/include/linux/vgaarb.h @@ -125,9 +125,11 @@ extern void vga_put(struct pci_dev *pdev, unsigned int rsrc); #ifdef CONFIG_VGA_ARB extern struct pci_dev *vga_default_device(void); extern void vga_set_default_device(struct pci_dev *pdev); +extern int vga_remove_vgacon(struct pci_dev *pdev); #else static inline struct pci_dev *vga_default_device(void) { return NULL; }; static inline void vga_set_default_device(struct pci_dev *pdev) { }; +static inline int vga_remove_vgacon(struct pci_dev *pdev) { return 0; }; #endif /* diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index c331e96a713b4..3e9ab643eedfe 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -382,6 +382,7 @@ void nft_unregister_set(struct nft_set_type *type); * @dtype: data type (verdict or numeric type defined by userspace) * @objtype: object type (see NFT_OBJECT_* definitions) * @size: maximum set size + * @use: number of rules references to this set * @nelems: number of elements * @ndeact: number of deactivated elements queued for removal * @timeout: default timeout value in jiffies @@ -407,6 +408,7 @@ struct nft_set { u32 dtype; u32 objtype; u32 size; + u32 use; atomic_t nelems; u32 ndeact; u64 timeout; @@ -416,7 +418,8 @@ struct nft_set { unsigned char *udata; /* runtime data below here */ const struct nft_set_ops *ops ____cacheline_aligned; - u16 flags:14, + u16 flags:13, + bound:1, genmask:2; u8 klen; u8 dlen; @@ -466,6 +469,10 @@ struct nft_set_binding { u32 flags; }; +enum nft_trans_phase; +void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set, + struct nft_set_binding *binding, + enum nft_trans_phase phase); int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set, struct nft_set_binding *binding); void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set, @@ -1344,15 +1351,12 @@ struct nft_trans_rule { struct nft_trans_set { struct nft_set *set; u32 set_id; - bool bound; }; #define nft_trans_set(trans) \ (((struct nft_trans_set *)trans->data)->set) #define nft_trans_set_id(trans) \ (((struct nft_trans_set *)trans->data)->set_id) -#define nft_trans_set_bound(trans) \ - (((struct nft_trans_set *)trans->data)->bound) struct nft_trans_chain { bool update; diff --git a/include/sound/pcm.h b/include/sound/pcm.h index 465d7d033c4c6..18bd8c3ea6053 100644 --- a/include/sound/pcm.h +++ b/include/sound/pcm.h @@ -750,7 +750,7 @@ static inline snd_pcm_uframes_t snd_pcm_playback_avail(struct snd_pcm_runtime *r } /** - * snd_pcm_playback_avail - Get the available (readable) space for capture + * snd_pcm_capture_avail - Get the available (readable) space for capture * @runtime: PCM runtime instance * * Result is between 0 ... (boundary - 1) diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index 3ec73f17ee2a4..a3916b4dd57ed 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -149,6 +149,17 @@ TRACE_DEFINE_ENUM(CP_TRIMMED); { CP_SPEC_LOG_NUM, "log type is 2" }, \ { CP_RECOVER_DIR, "dir needs recovery" }) +#define show_shutdown_mode(type) \ + __print_symbolic(type, \ + { F2FS_GOING_DOWN_FULLSYNC, "full sync" }, \ + { F2FS_GOING_DOWN_METASYNC, "meta sync" }, \ + { F2FS_GOING_DOWN_NOSYNC, "no sync" }, \ + { F2FS_GOING_DOWN_METAFLUSH, "meta flush" }, \ + { F2FS_GOING_DOWN_NEED_FSCK, "need fsck" }) + +struct f2fs_sb_info; +struct f2fs_io_info; +struct extent_info; struct victim_sel_policy; struct f2fs_map_blocks; @@ -533,6 +544,9 @@ TRACE_EVENT(f2fs_map_blocks, __field(block_t, m_lblk) __field(block_t, m_pblk) __field(unsigned int, m_len) + __field(unsigned int, m_flags) + __field(int, m_seg_type) + __field(bool, m_may_create) __field(int, ret) ), @@ -542,15 +556,22 @@ TRACE_EVENT(f2fs_map_blocks, __entry->m_lblk = map->m_lblk; __entry->m_pblk = map->m_pblk; __entry->m_len = map->m_len; + __entry->m_flags = map->m_flags; + __entry->m_seg_type = map->m_seg_type; + __entry->m_may_create = map->m_may_create; __entry->ret = ret; ), TP_printk("dev = (%d,%d), ino = %lu, file offset = %llu, " - "start blkaddr = 0x%llx, len = 0x%llx, err = %d", + "start blkaddr = 0x%llx, len = 0x%llx, flags = %u," + "seg_type = %d, may_create = %d, err = %d", show_dev_ino(__entry), (unsigned long long)__entry->m_lblk, (unsigned long long)__entry->m_pblk, (unsigned long long)__entry->m_len, + __entry->m_flags, + __entry->m_seg_type, + __entry->m_may_create, __entry->ret) ); @@ -1616,6 +1637,30 @@ DEFINE_EVENT(f2fs_sync_dirty_inodes, f2fs_sync_dirty_inodes_exit, TP_ARGS(sb, type, count) ); +TRACE_EVENT(f2fs_shutdown, + + TP_PROTO(struct f2fs_sb_info *sbi, unsigned int mode, int ret), + + TP_ARGS(sbi, mode, ret), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, mode) + __field(int, ret) + ), + + TP_fast_assign( + __entry->dev = sbi->sb->s_dev; + __entry->mode = mode; + __entry->ret = ret; + ), + + TP_printk("dev = (%d,%d), mode: %s, ret:%d", + show_dev(__entry->dev), + show_shutdown_mode(__entry->mode), + __entry->ret) +); + #endif /* _TRACE_F2FS_H */ /* This part must be outside protection */ diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h index 8451f30c6a0fe..7e899e635d33a 100644 --- a/include/trace/events/sunrpc.h +++ b/include/trace/events/sunrpc.h @@ -712,7 +712,8 @@ TRACE_EVENT(xprt_transmit, TP_fast_assign( __entry->task_id = rqst->rq_task->tk_pid; - __entry->client_id = rqst->rq_task->tk_client->cl_clid; + __entry->client_id = rqst->rq_task->tk_client ? + rqst->rq_task->tk_client->cl_clid : -1; __entry->xid = be32_to_cpu(rqst->rq_xid); __entry->seqno = rqst->rq_seqno; __entry->status = status; @@ -742,7 +743,8 @@ TRACE_EVENT(xprt_enq_xmit, TP_fast_assign( __entry->task_id = task->tk_pid; - __entry->client_id = task->tk_client->cl_clid; + __entry->client_id = task->tk_client ? + task->tk_client->cl_clid : -1; __entry->xid = be32_to_cpu(task->tk_rqstp->rq_xid); __entry->seqno = task->tk_rqstp->rq_seqno; __entry->stage = stage; diff --git a/include/trace/events/tegra_apb_dma.h b/include/trace/events/tegra_apb_dma.h new file mode 100644 index 0000000000000..0818f62861109 --- /dev/null +++ b/include/trace/events/tegra_apb_dma.h @@ -0,0 +1,61 @@ +#if !defined(_TRACE_TEGRA_APB_DMA_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_TEGRA_APM_DMA_H + +#include <linux/tracepoint.h> +#include <linux/dmaengine.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM tegra_apb_dma + +TRACE_EVENT(tegra_dma_tx_status, + TP_PROTO(struct dma_chan *dc, dma_cookie_t cookie, struct dma_tx_state *state), + TP_ARGS(dc, cookie, state), + TP_STRUCT__entry( + __string(chan, dev_name(&dc->dev->device)) + __field(dma_cookie_t, cookie) + __field(__u32, residue) + ), + TP_fast_assign( + __assign_str(chan, dev_name(&dc->dev->device)); + __entry->cookie = cookie; + __entry->residue = state ? state->residue : (u32)-1; + ), + TP_printk("channel %s: dma cookie %d, residue %u", + __get_str(chan), __entry->cookie, __entry->residue) +); + +TRACE_EVENT(tegra_dma_complete_cb, + TP_PROTO(struct dma_chan *dc, int count, void *ptr), + TP_ARGS(dc, count, ptr), + TP_STRUCT__entry( + __string(chan, dev_name(&dc->dev->device)) + __field(int, count) + __field(void *, ptr) + ), + TP_fast_assign( + __assign_str(chan, dev_name(&dc->dev->device)); + __entry->count = count; + __entry->ptr = ptr; + ), + TP_printk("channel %s: done %d, ptr %p", + __get_str(chan), __entry->count, __entry->ptr) +); + +TRACE_EVENT(tegra_dma_isr, + TP_PROTO(struct dma_chan *dc, int irq), + TP_ARGS(dc, irq), + TP_STRUCT__entry( + __string(chan, dev_name(&dc->dev->device)) + __field(int, irq) + ), + TP_fast_assign( + __assign_str(chan, dev_name(&dc->dev->device)); + __entry->irq = irq; + ), + TP_printk("%s: irq %d\n", __get_str(chan), __entry->irq) +); + +#endif /* _TRACE_TEGRADMA_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/include/uapi/asm-generic/socket.h b/include/uapi/asm-generic/socket.h index c8b430cb6dc4d..8c1391c89171a 100644 --- a/include/uapi/asm-generic/socket.h +++ b/include/uapi/asm-generic/socket.h @@ -2,8 +2,8 @@ #ifndef __ASM_GENERIC_SOCKET_H #define __ASM_GENERIC_SOCKET_H +#include <linux/posix_types.h> #include <asm/sockios.h> -#include <asm/bitsperlong.h> /* For setsockopt(2) */ #define SOL_SOCKET 1 diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index bf4624efe5e62..dee7292e1df6b 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -824,6 +824,8 @@ __SYSCALL(__NR_futex_time64, sys_futex) __SYSCALL(__NR_sched_rr_get_interval_time64, sys_sched_rr_get_interval) #endif +#define __NR_pidfd_send_signal 424 +__SYSCALL(__NR_pidfd_send_signal, sys_pidfd_send_signal) #define __NR_io_uring_setup 425 __SYSCALL(__NR_io_uring_setup, sys_io_uring_setup) #define __NR_io_uring_enter 426 diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 6b7654b8001fc..02ca827b8fac7 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -65,6 +65,7 @@ int console_printk[4] = { CONSOLE_LOGLEVEL_MIN, /* minimum_console_loglevel */ CONSOLE_LOGLEVEL_DEFAULT, /* default_console_loglevel */ }; +EXPORT_SYMBOL_GPL(console_printk); atomic_t ignore_console_lock_warning __read_mostly = ATOMIC_INIT(0); EXPORT_SYMBOL(ignore_console_lock_warning); diff --git a/kernel/resource.c b/kernel/resource.c index e81b17b53fa53..92190f62ebc53 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -382,7 +382,7 @@ static int __walk_iomem_res_desc(resource_size_t start, resource_size_t end, int (*func)(struct resource *, void *)) { struct resource res; - int ret = -1; + int ret = -EINVAL; while (start < end && !find_next_iomem_res(start, end, flags, desc, first_lvl, &res)) { @@ -452,6 +452,9 @@ int walk_mem_res(u64 start, u64 end, void *arg, * This function calls the @func callback against all memory ranges of type * System RAM which are marked as IORESOURCE_SYSTEM_RAM and IORESOUCE_BUSY. * It is to be used only for System RAM. + * + * This will find System RAM ranges that are children of top-level resources + * in addition to top-level System RAM resources. */ int walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages, void *arg, int (*func)(unsigned long, unsigned long, void *)) @@ -460,14 +463,14 @@ int walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages, unsigned long flags; struct resource res; unsigned long pfn, end_pfn; - int ret = -1; + int ret = -EINVAL; start = (u64) start_pfn << PAGE_SHIFT; end = ((u64)(start_pfn + nr_pages) << PAGE_SHIFT) - 1; flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; while (start < end && !find_next_iomem_res(start, end, flags, IORES_DESC_NONE, - true, &res)) { + false, &res)) { pfn = (res.start + PAGE_SIZE - 1) >> PAGE_SHIFT; end_pfn = (res.end + 1) >> PAGE_SHIFT; if (end_pfn > pfn) @@ -1128,6 +1131,15 @@ struct resource * __request_region(struct resource *parent, conflict = __request_resource(parent, res); if (!conflict) break; + /* + * mm/hmm.c reserves physical addresses which then + * become unavailable to other users. Conflicts are + * not expected. Warn to aid debugging if encountered. + */ + if (conflict->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY) { + pr_warn("Unaddressable device %s %pR conflicts with %pR", + conflict->name, conflict, res); + } if (conflict != parent) { if (!(conflict->flags & IORESOURCE_BUSY)) { parent = conflict; diff --git a/kernel/signal.c b/kernel/signal.c index 5d53183e27057..b7953934aa994 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -19,7 +19,9 @@ #include <linux/sched/task.h> #include <linux/sched/task_stack.h> #include <linux/sched/cputime.h> +#include <linux/file.h> #include <linux/fs.h> +#include <linux/proc_fs.h> #include <linux/tty.h> #include <linux/binfmts.h> #include <linux/coredump.h> @@ -3487,6 +3489,16 @@ COMPAT_SYSCALL_DEFINE4(rt_sigtimedwait_time32, compat_sigset_t __user *, uthese, #endif #endif +static inline void prepare_kill_siginfo(int sig, struct kernel_siginfo *info) +{ + clear_siginfo(info); + info->si_signo = sig; + info->si_errno = 0; + info->si_code = SI_USER; + info->si_pid = task_tgid_vnr(current); + info->si_uid = from_kuid_munged(current_user_ns(), current_uid()); +} + /** * sys_kill - send a signal to a process * @pid: the PID of the process @@ -3496,16 +3508,125 @@ SYSCALL_DEFINE2(kill, pid_t, pid, int, sig) { struct kernel_siginfo info; - clear_siginfo(&info); - info.si_signo = sig; - info.si_errno = 0; - info.si_code = SI_USER; - info.si_pid = task_tgid_vnr(current); - info.si_uid = from_kuid_munged(current_user_ns(), current_uid()); + prepare_kill_siginfo(sig, &info); return kill_something_info(sig, &info, pid); } +#ifdef CONFIG_PROC_FS +/* + * Verify that the signaler and signalee either are in the same pid namespace + * or that the signaler's pid namespace is an ancestor of the signalee's pid + * namespace. + */ +static bool access_pidfd_pidns(struct pid *pid) +{ + struct pid_namespace *active = task_active_pid_ns(current); + struct pid_namespace *p = ns_of_pid(pid); + + for (;;) { + if (!p) + return false; + if (p == active) + break; + p = p->parent; + } + + return true; +} + +static int copy_siginfo_from_user_any(kernel_siginfo_t *kinfo, siginfo_t *info) +{ +#ifdef CONFIG_COMPAT + /* + * Avoid hooking up compat syscalls and instead handle necessary + * conversions here. Note, this is a stop-gap measure and should not be + * considered a generic solution. + */ + if (in_compat_syscall()) + return copy_siginfo_from_user32( + kinfo, (struct compat_siginfo __user *)info); +#endif + return copy_siginfo_from_user(kinfo, info); +} + +/** + * sys_pidfd_send_signal - send a signal to a process through a task file + * descriptor + * @pidfd: the file descriptor of the process + * @sig: signal to be sent + * @info: the signal info + * @flags: future flags to be passed + * + * The syscall currently only signals via PIDTYPE_PID which covers + * kill(<positive-pid>, <signal>. It does not signal threads or process + * groups. + * In order to extend the syscall to threads and process groups the @flags + * argument should be used. In essence, the @flags argument will determine + * what is signaled and not the file descriptor itself. Put in other words, + * grouping is a property of the flags argument not a property of the file + * descriptor. + * + * Return: 0 on success, negative errno on failure + */ +SYSCALL_DEFINE4(pidfd_send_signal, int, pidfd, int, sig, + siginfo_t __user *, info, unsigned int, flags) +{ + int ret; + struct fd f; + struct pid *pid; + kernel_siginfo_t kinfo; + + /* Enforce flags be set to 0 until we add an extension. */ + if (flags) + return -EINVAL; + + f = fdget_raw(pidfd); + if (!f.file) + return -EBADF; + + /* Is this a pidfd? */ + pid = tgid_pidfd_to_pid(f.file); + if (IS_ERR(pid)) { + ret = PTR_ERR(pid); + goto err; + } + + ret = -EINVAL; + if (!access_pidfd_pidns(pid)) + goto err; + + if (info) { + ret = copy_siginfo_from_user_any(&kinfo, info); + if (unlikely(ret)) + goto err; + + ret = -EINVAL; + if (unlikely(sig != kinfo.si_signo)) + goto err; + + if ((task_pid(current) != pid) && + (kinfo.si_code >= 0 || kinfo.si_code == SI_TKILL)) { + /* Only allow sending arbitrary signals to yourself. */ + ret = -EPERM; + if (kinfo.si_code != SI_USER) + goto err; + + /* Turn this into a regular kill signal. */ + prepare_kill_siginfo(sig, &kinfo); + } + } else { + prepare_kill_siginfo(sig, &kinfo); + } + + ret = kill_pid_info(sig, &kinfo, pid); + +err: + fdput(f); + return ret; +} +#endif /* CONFIG_PROC_FS */ + static int do_send_specific(pid_t tgid, pid_t pid, int sig, struct kernel_siginfo *info) { diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index 51d7c6794bf11..d21f4befaea4d 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -168,6 +168,7 @@ COND_SYSCALL(syslog); /* kernel/sched/core.c */ /* kernel/signal.c */ +COND_SYSCALL(pidfd_send_signal); /* kernel/sys.c */ COND_SYSCALL(setregid); diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index fac0ddf8a8e22..e1c6d79fb4cc9 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -723,6 +723,7 @@ int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg) #endif case BLKTRACESTART: start = 1; + /* fall through */ case BLKTRACESTOP: ret = __blk_trace_startstop(q, start); break; diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 9a91479bbbfe1..41b6f96e53662 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -4191,6 +4191,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_consume); * ring_buffer_read_prepare - Prepare for a non consuming read of the buffer * @buffer: The ring buffer to read from * @cpu: The cpu buffer to iterate over + * @flags: gfp flags to use for memory allocation * * This performs the initial preparations necessary to iterate * through the buffer. Memory is allocated, buffer recording @@ -4208,7 +4209,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_consume); * This overall must be paired with ring_buffer_read_finish. */ struct ring_buffer_iter * -ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu) +ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu, gfp_t flags) { struct ring_buffer_per_cpu *cpu_buffer; struct ring_buffer_iter *iter; @@ -4216,7 +4217,7 @@ ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu) if (!cpumask_test_cpu(cpu, buffer->cpumask)) return NULL; - iter = kmalloc(sizeof(*iter), GFP_KERNEL); + iter = kmalloc(sizeof(*iter), flags); if (!iter) return NULL; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 94ffc1c559d51..21153e64bf1c3 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4079,7 +4079,8 @@ __tracing_open(struct inode *inode, struct file *file, bool snapshot) if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { for_each_tracing_cpu(cpu) { iter->buffer_iter[cpu] = - ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu); + ring_buffer_read_prepare(iter->trace_buffer->buffer, + cpu, GFP_KERNEL); } ring_buffer_read_prepare_sync(); for_each_tracing_cpu(cpu) { @@ -4089,7 +4090,8 @@ __tracing_open(struct inode *inode, struct file *file, bool snapshot) } else { cpu = iter->cpu_file; iter->buffer_iter[cpu] = - ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu); + ring_buffer_read_prepare(iter->trace_buffer->buffer, + cpu, GFP_KERNEL); ring_buffer_read_prepare_sync(); ring_buffer_read_start(iter->buffer_iter[cpu]); tracing_iter_reset(iter, cpu); diff --git a/kernel/trace/trace_kdb.c b/kernel/trace/trace_kdb.c index d953c163a0794..810d78a8d14c7 100644 --- a/kernel/trace/trace_kdb.c +++ b/kernel/trace/trace_kdb.c @@ -51,14 +51,16 @@ static void ftrace_dump_buf(int skip_lines, long cpu_file) if (cpu_file == RING_BUFFER_ALL_CPUS) { for_each_tracing_cpu(cpu) { iter.buffer_iter[cpu] = - ring_buffer_read_prepare(iter.trace_buffer->buffer, cpu); + ring_buffer_read_prepare(iter.trace_buffer->buffer, + cpu, GFP_ATOMIC); ring_buffer_read_start(iter.buffer_iter[cpu]); tracing_iter_reset(&iter, cpu); } } else { iter.cpu_file = cpu_file; iter.buffer_iter[cpu_file] = - ring_buffer_read_prepare(iter.trace_buffer->buffer, cpu_file); + ring_buffer_read_prepare(iter.trace_buffer->buffer, + cpu_file, GFP_ATOMIC); ring_buffer_read_start(iter.buffer_iter[cpu_file]); tracing_iter_reset(&iter, cpu_file); } diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 99592c27465e1..5d5129b05df78 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -35,7 +35,7 @@ static struct dyn_event_operations trace_kprobe_ops = { .match = trace_kprobe_match, }; -/** +/* * Kprobe event core functions */ struct trace_kprobe { @@ -221,7 +221,7 @@ static struct trace_kprobe *alloc_trace_kprobe(const char *group, tk->rp.maxactive = maxactive; - if (!event || !is_good_name(event)) { + if (!event || !group) { ret = -EINVAL; goto error; } @@ -231,11 +231,6 @@ static struct trace_kprobe *alloc_trace_kprobe(const char *group, if (!tk->tp.call.name) goto error; - if (!group || !is_good_name(group)) { - ret = -EINVAL; - goto error; - } - tk->tp.class.system = kstrdup(group, GFP_KERNEL); if (!tk->tp.class.system) goto error; @@ -624,7 +619,11 @@ static int trace_kprobe_create(int argc, const char *argv[]) if (event) event++; - if (is_return && isdigit(argv[0][1])) { + if (isdigit(argv[0][1])) { + if (!is_return) { + pr_info("Maxactive is not for kprobe"); + return -EINVAL; + } if (event) len = event - &argv[0][1] - 1; else @@ -634,8 +633,8 @@ static int trace_kprobe_create(int argc, const char *argv[]) memcpy(buf, &argv[0][1], len); buf[len] = '\0'; ret = kstrtouint(buf, 0, &maxactive); - if (ret) { - pr_info("Failed to parse maxactive.\n"); + if (ret || !maxactive) { + pr_info("Invalid maxactive number\n"); return ret; } /* kretprobes instances are iterated over via a list. The @@ -694,9 +693,9 @@ static int trace_kprobe_create(int argc, const char *argv[]) tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, argc, is_return); if (IS_ERR(tk)) { - pr_info("Failed to allocate trace_probe.(%d)\n", - (int)PTR_ERR(tk)); ret = PTR_ERR(tk); + /* This must return -ENOMEM otherwise there is a bug */ + WARN_ON_ONCE(ret != -ENOMEM); goto out; } diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 89da34b326e3c..8f8411e7835fd 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -13,7 +13,7 @@ #include "trace_probe.h" -const char *reserved_field_names[] = { +static const char *reserved_field_names[] = { "common_type", "common_flags", "common_preempt_count", @@ -159,6 +159,7 @@ int traceprobe_parse_event_name(const char **pevent, const char **pgroup, char *buf) { const char *slash, *event = *pevent; + int len; slash = strchr(event, '/'); if (slash) { @@ -171,12 +172,25 @@ int traceprobe_parse_event_name(const char **pevent, const char **pgroup, return -E2BIG; } strlcpy(buf, event, slash - event + 1); + if (!is_good_name(buf)) { + pr_info("Group name must follow the same rules as C identifiers\n"); + return -EINVAL; + } *pgroup = buf; *pevent = slash + 1; + event = *pevent; } - if (strlen(event) == 0) { + len = strlen(event); + if (len == 0) { pr_info("Event name is not specified\n"); return -EINVAL; + } else if (len > MAX_EVENT_NAME_LEN) { + pr_info("Event name is too long\n"); + return -E2BIG; + } + if (!is_good_name(event)) { + pr_info("Event name must follow the same rules as C identifiers\n"); + return -EINVAL; } return 0; } @@ -548,6 +562,8 @@ int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, char *arg, body = strchr(arg, '='); if (body) { + if (body - arg > MAX_ARG_NAME_LEN || body == arg) + return -EINVAL; parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL); body++; } else { diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 8a63f8bc01bc7..2177c206de151 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -32,6 +32,7 @@ #define MAX_TRACE_ARGS 128 #define MAX_ARGSTR_LEN 63 #define MAX_ARRAY_LEN 64 +#define MAX_ARG_NAME_LEN 32 #define MAX_STRING_SIZE PATH_MAX /* Reserved field names */ diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 9bde07c06362f..be78d99ee6bc2 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -273,10 +273,7 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret) { struct trace_uprobe *tu; - if (!event || !is_good_name(event)) - return ERR_PTR(-EINVAL); - - if (!group || !is_good_name(group)) + if (!event || !group) return ERR_PTR(-EINVAL); tu = kzalloc(SIZEOF_TRACE_UPROBE(nargs), GFP_KERNEL); @@ -524,8 +521,9 @@ static int trace_uprobe_create(int argc, const char **argv) tu = alloc_trace_uprobe(group, event, argc, is_return); if (IS_ERR(tu)) { - pr_info("Failed to allocate trace_uprobe.(%d)\n", (int)PTR_ERR(tu)); ret = PTR_ERR(tu); + /* This must return -ENOMEM otherwise there is a bug */ + WARN_ON_ONCE(ret != -ENOMEM); goto fail_address_parse; } tu->offset = offset; diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile index 4e90d443d1b09..e723eacf7868d 100644 --- a/lib/raid6/Makefile +++ b/lib/raid6/Makefile @@ -39,7 +39,7 @@ endif ifeq ($(CONFIG_KERNEL_MODE_NEON),y) NEON_FLAGS := -ffreestanding ifeq ($(ARCH),arm) -NEON_FLAGS += -mfloat-abi=softfp -mfpu=neon +NEON_FLAGS += -march=armv7-a -mfloat-abi=softfp -mfpu=neon endif CFLAGS_recov_neon_inner.o += $(NEON_FLAGS) ifeq ($(ARCH),arm64) diff --git a/mm/filemap.c b/mm/filemap.c index a3b4021c448fb..d78f577baef2a 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1587,6 +1587,9 @@ EXPORT_SYMBOL(find_lock_entry); * @gfp_mask and added to the page cache and the VM's LRU * list. The page is returned locked and with an increased * refcount. + * - FGP_FOR_MMAP: Similar to FGP_CREAT, only we want to allow the caller to do + * its own locking dance if the page is already in cache, or unlock the page + * before returning if we had to add the page to pagecache. * * If FGP_LOCK or FGP_CREAT are specified then the function may sleep even * if the GFP flags specified for FGP_CREAT are atomic. @@ -1641,7 +1644,7 @@ no_page: if (!page) return NULL; - if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK))) + if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP)))) fgp_flags |= FGP_LOCK; /* Init accessed so avoid atomic mark_page_accessed later */ @@ -1655,6 +1658,13 @@ no_page: if (err == -EEXIST) goto repeat; } + + /* + * add_to_page_cache_lru locks the page, and for mmap we expect + * an unlocked page. + */ + if (page && (fgp_flags & FGP_FOR_MMAP)) + unlock_page(page); } return page; @@ -2379,64 +2389,98 @@ out: EXPORT_SYMBOL(generic_file_read_iter); #ifdef CONFIG_MMU -/** - * page_cache_read - adds requested page to the page cache if not already there - * @file: file to read - * @offset: page index - * @gfp_mask: memory allocation flags - * - * This adds the requested page to the page cache if it isn't already there, - * and schedules an I/O to read in its contents from disk. - * - * Return: %0 on success, negative error code otherwise. - */ -static int page_cache_read(struct file *file, pgoff_t offset, gfp_t gfp_mask) +#define MMAP_LOTSAMISS (100) +static struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf, + struct file *fpin) { - struct address_space *mapping = file->f_mapping; - struct page *page; - int ret; + int flags = vmf->flags; - do { - page = __page_cache_alloc(gfp_mask); - if (!page) - return -ENOMEM; + if (fpin) + return fpin; - ret = add_to_page_cache_lru(page, mapping, offset, gfp_mask); - if (ret == 0) - ret = mapping->a_ops->readpage(file, page); - else if (ret == -EEXIST) - ret = 0; /* losing race to add is OK */ + /* + * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or + * anything, so we only pin the file and drop the mmap_sem if only + * FAULT_FLAG_ALLOW_RETRY is set. + */ + if ((flags & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT)) == + FAULT_FLAG_ALLOW_RETRY) { + fpin = get_file(vmf->vma->vm_file); + up_read(&vmf->vma->vm_mm->mmap_sem); + } + return fpin; +} - put_page(page); +/* + * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_sem + * @vmf - the vm_fault for this fault. + * @page - the page to lock. + * @fpin - the pointer to the file we may pin (or is already pinned). + * + * This works similar to lock_page_or_retry in that it can drop the mmap_sem. + * It differs in that it actually returns the page locked if it returns 1 and 0 + * if it couldn't lock the page. If we did have to drop the mmap_sem then fpin + * will point to the pinned file and needs to be fput()'ed at a later point. + */ +static int lock_page_maybe_drop_mmap(struct vm_fault *vmf, struct page *page, + struct file **fpin) +{ + if (trylock_page(page)) + return 1; - } while (ret == AOP_TRUNCATED_PAGE); + /* + * NOTE! This will make us return with VM_FAULT_RETRY, but with + * the mmap_sem still held. That's how FAULT_FLAG_RETRY_NOWAIT + * is supposed to work. We have way too many special cases.. + */ + if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) + return 0; - return ret; + *fpin = maybe_unlock_mmap_for_io(vmf, *fpin); + if (vmf->flags & FAULT_FLAG_KILLABLE) { + if (__lock_page_killable(page)) { + /* + * We didn't have the right flags to drop the mmap_sem, + * but all fault_handlers only check for fatal signals + * if we return VM_FAULT_RETRY, so we need to drop the + * mmap_sem here and return 0 if we don't have a fpin. + */ + if (*fpin == NULL) + up_read(&vmf->vma->vm_mm->mmap_sem); + return 0; + } + } else + __lock_page(page); + return 1; } -#define MMAP_LOTSAMISS (100) /* - * Synchronous readahead happens when we don't even find - * a page in the page cache at all. + * Synchronous readahead happens when we don't even find a page in the page + * cache at all. We don't want to perform IO under the mmap sem, so if we have + * to drop the mmap sem we return the file that was pinned in order for us to do + * that. If we didn't pin a file then we return NULL. The file that is + * returned needs to be fput()'ed when we're done with it. */ -static void do_sync_mmap_readahead(struct vm_area_struct *vma, - struct file_ra_state *ra, - struct file *file, - pgoff_t offset) +static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) { + struct file *file = vmf->vma->vm_file; + struct file_ra_state *ra = &file->f_ra; struct address_space *mapping = file->f_mapping; + struct file *fpin = NULL; + pgoff_t offset = vmf->pgoff; /* If we don't want any read-ahead, don't bother */ - if (vma->vm_flags & VM_RAND_READ) - return; + if (vmf->vma->vm_flags & VM_RAND_READ) + return fpin; if (!ra->ra_pages) - return; + return fpin; - if (vma->vm_flags & VM_SEQ_READ) { + if (vmf->vma->vm_flags & VM_SEQ_READ) { + fpin = maybe_unlock_mmap_for_io(vmf, fpin); page_cache_sync_readahead(mapping, ra, file, offset, ra->ra_pages); - return; + return fpin; } /* Avoid banging the cache line if not needed */ @@ -2448,37 +2492,44 @@ static void do_sync_mmap_readahead(struct vm_area_struct *vma, * stop bothering with read-ahead. It will only hurt. */ if (ra->mmap_miss > MMAP_LOTSAMISS) - return; + return fpin; /* * mmap read-around */ + fpin = maybe_unlock_mmap_for_io(vmf, fpin); ra->start = max_t(long, 0, offset - ra->ra_pages / 2); ra->size = ra->ra_pages; ra->async_size = ra->ra_pages / 4; ra_submit(ra, mapping, file); + return fpin; } /* * Asynchronous readahead happens when we find the page and PG_readahead, - * so we want to possibly extend the readahead further.. + * so we want to possibly extend the readahead further. We return the file that + * was pinned if we have to drop the mmap_sem in order to do IO. */ -static void do_async_mmap_readahead(struct vm_area_struct *vma, - struct file_ra_state *ra, - struct file *file, - struct page *page, - pgoff_t offset) +static struct file *do_async_mmap_readahead(struct vm_fault *vmf, + struct page *page) { + struct file *file = vmf->vma->vm_file; + struct file_ra_state *ra = &file->f_ra; struct address_space *mapping = file->f_mapping; + struct file *fpin = NULL; + pgoff_t offset = vmf->pgoff; /* If we don't want any read-ahead, don't bother */ - if (vma->vm_flags & VM_RAND_READ) - return; + if (vmf->vma->vm_flags & VM_RAND_READ) + return fpin; if (ra->mmap_miss > 0) ra->mmap_miss--; - if (PageReadahead(page)) + if (PageReadahead(page)) { + fpin = maybe_unlock_mmap_for_io(vmf, fpin); page_cache_async_readahead(mapping, ra, file, page, offset, ra->ra_pages); + } + return fpin; } /** @@ -2510,6 +2561,7 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) { int error; struct file *file = vmf->vma->vm_file; + struct file *fpin = NULL; struct address_space *mapping = file->f_mapping; struct file_ra_state *ra = &file->f_ra; struct inode *inode = mapping->host; @@ -2531,23 +2583,26 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) * We found the page, so try async readahead before * waiting for the lock. */ - do_async_mmap_readahead(vmf->vma, ra, file, page, offset); + fpin = do_async_mmap_readahead(vmf, page); } else if (!page) { /* No page in the page cache at all */ - do_sync_mmap_readahead(vmf->vma, ra, file, offset); count_vm_event(PGMAJFAULT); count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT); ret = VM_FAULT_MAJOR; + fpin = do_sync_mmap_readahead(vmf); retry_find: - page = find_get_page(mapping, offset); - if (!page) - goto no_cached_page; + page = pagecache_get_page(mapping, offset, + FGP_CREAT|FGP_FOR_MMAP, + vmf->gfp_mask); + if (!page) { + if (fpin) + goto out_retry; + return vmf_error(-ENOMEM); + } } - if (!lock_page_or_retry(page, vmf->vma->vm_mm, vmf->flags)) { - put_page(page); - return ret | VM_FAULT_RETRY; - } + if (!lock_page_maybe_drop_mmap(vmf, page, &fpin)) + goto out_retry; /* Did it get truncated? */ if (unlikely(page->mapping != mapping)) { @@ -2565,6 +2620,16 @@ retry_find: goto page_not_uptodate; /* + * We've made it this far and we had to drop our mmap_sem, now is the + * time to return to the upper layer and have it re-find the vma and + * redo the fault. + */ + if (fpin) { + unlock_page(page); + goto out_retry; + } + + /* * Found the page and have a reference on it. * We must recheck i_size under page lock. */ @@ -2578,28 +2643,6 @@ retry_find: vmf->page = page; return ret | VM_FAULT_LOCKED; -no_cached_page: - /* - * We're only likely to ever get here if MADV_RANDOM is in - * effect. - */ - error = page_cache_read(file, offset, vmf->gfp_mask); - - /* - * The page we want has now been added to the page cache. - * In the unlikely event that someone removed it in the - * meantime, we'll just come back here and read it again. - */ - if (error >= 0) - goto retry_find; - - /* - * An error return from page_cache_read can result if the - * system is low on memory, or a problem occurs while trying - * to schedule I/O. - */ - return vmf_error(error); - page_not_uptodate: /* * Umm, take care of errors if the page isn't up-to-date. @@ -2608,12 +2651,15 @@ page_not_uptodate: * and we need to check for errors. */ ClearPageError(page); + fpin = maybe_unlock_mmap_for_io(vmf, fpin); error = mapping->a_ops->readpage(file, page); if (!error) { wait_on_page_locked(page); if (!PageUptodate(page)) error = -EIO; } + if (fpin) + goto out_retry; put_page(page); if (!error || error == AOP_TRUNCATED_PAGE) @@ -2622,6 +2668,18 @@ page_not_uptodate: /* Things didn't work out. Return zero to tell the mm layer so. */ shrink_readahead_size_eio(file, ra); return VM_FAULT_SIGBUS; + +out_retry: + /* + * We dropped the mmap_sem, we need to return to the fault handler to + * re-find the vma and come back and find our hopefully still populated + * page. + */ + if (page) + put_page(page); + if (fpin) + fput(fpin); + return ret | VM_FAULT_RETRY; } EXPORT_SYMBOL(filemap_fault); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index cd23c081924de..f767582af4f8c 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -101,28 +101,24 @@ u64 max_mem_size = U64_MAX; /* add this memory to iomem resource */ static struct resource *register_memory_resource(u64 start, u64 size) { - struct resource *res, *conflict; + struct resource *res; + unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + char *resource_name = "System RAM"; if (start + size > max_mem_size) return ERR_PTR(-E2BIG); - res = kzalloc(sizeof(struct resource), GFP_KERNEL); - if (!res) - return ERR_PTR(-ENOMEM); - - res->name = "System RAM"; - res->start = start; - res->end = start + size - 1; - res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; - conflict = request_resource_conflict(&iomem_resource, res); - if (conflict) { - if (conflict->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY) { - pr_debug("Device unaddressable memory block " - "memory hotplug at %#010llx !\n", - (unsigned long long)start); - } - pr_debug("System RAM resource %pR cannot be added\n", res); - kfree(res); + /* + * Request ownership of the new memory range. This might be + * a child of an existing resource that was present but + * not marked as busy. + */ + res = __request_region(&iomem_resource, start, size, + resource_name, flags); + + if (!res) { + pr_debug("Unable to reserve System RAM region: %016llx->%016llx\n", + start, start + size); return ERR_PTR(-EEXIST); } return res; diff --git a/net/9p/client.c b/net/9p/client.c index 357214a51f138..b85d51f4b8ebe 100644 --- a/net/9p/client.c +++ b/net/9p/client.c @@ -1061,7 +1061,7 @@ struct p9_client *p9_client_create(const char *dev_name, char *options) p9_debug(P9_DEBUG_ERROR, "Please specify a msize of at least 4k\n"); err = -EINVAL; - goto free_client; + goto close_trans; } err = p9_client_version(clnt); diff --git a/net/9p/trans_xen.c b/net/9p/trans_xen.c index e2fbf3677b9ba..29420ebb8f070 100644 --- a/net/9p/trans_xen.c +++ b/net/9p/trans_xen.c @@ -513,7 +513,7 @@ static void xen_9pfs_front_changed(struct xenbus_device *dev, case XenbusStateClosed: if (dev->state == XenbusStateClosed) break; - /* Missed the backend's CLOSING state -- fallthrough */ + /* fall through - Missed the backend's CLOSING state */ case XenbusStateClosing: xenbus_frontend_closed(dev); break; diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 831d844a27ca2..277d71239d755 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1734,15 +1734,8 @@ EXPORT_SYMBOL(tcp_add_backlog); int tcp_filter(struct sock *sk, struct sk_buff *skb) { struct tcphdr *th = (struct tcphdr *)skb->data; - unsigned int eaten = skb->len; - int err; - err = sk_filter_trim_cap(sk, skb, th->doff * 4); - if (!err) { - eaten -= skb->len; - TCP_SKB_CB(skb)->end_seq -= eaten; - } - return err; + return sk_filter_trim_cap(sk, skb, th->doff * 4); } EXPORT_SYMBOL(tcp_filter); diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index 09e440e8dfaec..07e21a82ce4cc 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -778,8 +778,9 @@ static bool check_6rd(struct ip_tunnel *tunnel, const struct in6_addr *v6dst, pbw0 = tunnel->ip6rd.prefixlen >> 5; pbi0 = tunnel->ip6rd.prefixlen & 0x1f; - d = (ntohl(v6dst->s6_addr32[pbw0]) << pbi0) >> - tunnel->ip6rd.relay_prefixlen; + d = tunnel->ip6rd.relay_prefixlen < 32 ? + (ntohl(v6dst->s6_addr32[pbw0]) << pbi0) >> + tunnel->ip6rd.relay_prefixlen : 0; pbi1 = pbi0 - tunnel->ip6rd.relay_prefixlen; if (pbi1 > 0) diff --git a/net/l2tp/l2tp_ip6.c b/net/l2tp/l2tp_ip6.c index 0ae6899edac07..37a69df17cab9 100644 --- a/net/l2tp/l2tp_ip6.c +++ b/net/l2tp/l2tp_ip6.c @@ -674,9 +674,6 @@ static int l2tp_ip6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, if (flags & MSG_OOB) goto out; - if (addr_len) - *addr_len = sizeof(*lsa); - if (flags & MSG_ERRQUEUE) return ipv6_recv_error(sk, msg, len, addr_len); @@ -706,6 +703,7 @@ static int l2tp_ip6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, lsa->l2tp_conn_id = 0; if (ipv6_addr_type(&lsa->l2tp_addr) & IPV6_ADDR_LINKLOCAL) lsa->l2tp_scope_id = inet6_iif(skb); + *addr_len = sizeof(*lsa); } if (np->rxopt.all) diff --git a/net/netfilter/nf_nat_masquerade.c b/net/netfilter/nf_nat_masquerade.c index 86fa4dcc63c55..d85c4d902e7b1 100644 --- a/net/netfilter/nf_nat_masquerade.c +++ b/net/netfilter/nf_nat_masquerade.c @@ -11,7 +11,8 @@ #include <net/netfilter/ipv6/nf_nat_masquerade.h> static DEFINE_MUTEX(masq_mutex); -static unsigned int masq_refcnt __read_mostly; +static unsigned int masq_refcnt4 __read_mostly; +static unsigned int masq_refcnt6 __read_mostly; unsigned int nf_nat_masquerade_ipv4(struct sk_buff *skb, unsigned int hooknum, @@ -141,8 +142,13 @@ int nf_nat_masquerade_ipv4_register_notifier(void) int ret = 0; mutex_lock(&masq_mutex); + if (WARN_ON_ONCE(masq_refcnt4 == UINT_MAX)) { + ret = -EOVERFLOW; + goto out_unlock; + } + /* check if the notifier was already set */ - if (++masq_refcnt > 1) + if (++masq_refcnt4 > 1) goto out_unlock; /* Register for device down reports */ @@ -160,7 +166,7 @@ int nf_nat_masquerade_ipv4_register_notifier(void) err_unregister: unregister_netdevice_notifier(&masq_dev_notifier); err_dec: - masq_refcnt--; + masq_refcnt4--; out_unlock: mutex_unlock(&masq_mutex); return ret; @@ -171,7 +177,7 @@ void nf_nat_masquerade_ipv4_unregister_notifier(void) { mutex_lock(&masq_mutex); /* check if the notifier still has clients */ - if (--masq_refcnt > 0) + if (--masq_refcnt4 > 0) goto out_unlock; unregister_netdevice_notifier(&masq_dev_notifier); @@ -321,25 +327,23 @@ int nf_nat_masquerade_ipv6_register_notifier(void) int ret = 0; mutex_lock(&masq_mutex); - /* check if the notifier is already set */ - if (++masq_refcnt > 1) + if (WARN_ON_ONCE(masq_refcnt6 == UINT_MAX)) { + ret = -EOVERFLOW; goto out_unlock; + } - ret = register_netdevice_notifier(&masq_dev_notifier); - if (ret) - goto err_dec; + /* check if the notifier is already set */ + if (++masq_refcnt6 > 1) + goto out_unlock; ret = register_inet6addr_notifier(&masq_inet6_notifier); if (ret) - goto err_unregister; + goto err_dec; mutex_unlock(&masq_mutex); return ret; - -err_unregister: - unregister_netdevice_notifier(&masq_dev_notifier); err_dec: - masq_refcnt--; + masq_refcnt6--; out_unlock: mutex_unlock(&masq_mutex); return ret; @@ -350,11 +354,10 @@ void nf_nat_masquerade_ipv6_unregister_notifier(void) { mutex_lock(&masq_mutex); /* check if the notifier still has clients */ - if (--masq_refcnt > 0) + if (--masq_refcnt6 > 0) goto out_unlock; unregister_inet6addr_notifier(&masq_inet6_notifier); - unregister_netdevice_notifier(&masq_dev_notifier); out_unlock: mutex_unlock(&masq_mutex); } diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index faf6bd10a19f9..513f931186043 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -142,7 +142,7 @@ static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set) list_for_each_entry_reverse(trans, &net->nft.commit_list, list) { if (trans->msg_type == NFT_MSG_NEWSET && nft_trans_set(trans) == set) { - nft_trans_set_bound(trans) = true; + set->bound = true; break; } } @@ -2162,9 +2162,11 @@ err1: static void nf_tables_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr) { + const struct nft_expr_type *type = expr->ops->type; + if (expr->ops->destroy) expr->ops->destroy(ctx, expr); - module_put(expr->ops->type->owner); + module_put(type->owner); } struct nft_expr *nft_expr_init(const struct nft_ctx *ctx, @@ -3672,6 +3674,9 @@ err1: static void nft_set_destroy(struct nft_set *set) { + if (WARN_ON(set->use > 0)) + return; + set->ops->destroy(set); module_put(to_set_type(set->ops)->owner); kfree(set->name); @@ -3712,7 +3717,7 @@ static int nf_tables_delset(struct net *net, struct sock *nlsk, NL_SET_BAD_ATTR(extack, attr); return PTR_ERR(set); } - if (!list_empty(&set->bindings) || + if (set->use || (nlh->nlmsg_flags & NLM_F_NONREC && atomic_read(&set->nelems) > 0)) { NL_SET_BAD_ATTR(extack, attr); return -EBUSY; @@ -3742,6 +3747,9 @@ int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set, struct nft_set_binding *i; struct nft_set_iter iter; + if (set->use == UINT_MAX) + return -EOVERFLOW; + if (!list_empty(&set->bindings) && nft_set_is_anonymous(set)) return -EBUSY; @@ -3769,6 +3777,7 @@ bind: binding->chain = ctx->chain; list_add_tail_rcu(&binding->list, &set->bindings); nft_set_trans_bind(ctx, set); + set->use++; return 0; } @@ -3788,6 +3797,25 @@ void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set, } EXPORT_SYMBOL_GPL(nf_tables_unbind_set); +void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set, + struct nft_set_binding *binding, + enum nft_trans_phase phase) +{ + switch (phase) { + case NFT_TRANS_PREPARE: + set->use--; + return; + case NFT_TRANS_ABORT: + case NFT_TRANS_RELEASE: + set->use--; + /* fall through */ + default: + nf_tables_unbind_set(ctx, set, binding, + phase == NFT_TRANS_COMMIT); + } +} +EXPORT_SYMBOL_GPL(nf_tables_deactivate_set); + void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set) { if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) @@ -6536,6 +6564,11 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) struct nft_chain *chain; struct nft_table *table; + if (list_empty(&net->nft.commit_list)) { + mutex_unlock(&net->nft.commit_mutex); + return 0; + } + /* 0. Validate ruleset, otherwise roll back for error reporting. */ if (nf_tables_validate(net) < 0) return -EAGAIN; @@ -6709,8 +6742,7 @@ static void nf_tables_abort_release(struct nft_trans *trans) nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans)); break; case NFT_MSG_NEWSET: - if (!nft_trans_set_bound(trans)) - nft_set_destroy(nft_trans_set(trans)); + nft_set_destroy(nft_trans_set(trans)); break; case NFT_MSG_NEWSETELEM: nft_set_elem_destroy(nft_trans_elem_set(trans), @@ -6783,8 +6815,11 @@ static int __nf_tables_abort(struct net *net) break; case NFT_MSG_NEWSET: trans->ctx.table->use--; - if (!nft_trans_set_bound(trans)) - list_del_rcu(&nft_trans_set(trans)->list); + if (nft_trans_set(trans)->bound) { + nft_trans_destroy(trans); + break; + } + list_del_rcu(&nft_trans_set(trans)->list); break; case NFT_MSG_DELSET: trans->ctx.table->use++; @@ -6792,8 +6827,11 @@ static int __nf_tables_abort(struct net *net) nft_trans_destroy(trans); break; case NFT_MSG_NEWSETELEM: + if (nft_trans_elem_set(trans)->bound) { + nft_trans_destroy(trans); + break; + } te = (struct nft_trans_elem *)trans->data; - te->set->ops->remove(net, te->set, &te->elem); atomic_dec(&te->set->nelems); break; diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c index a8a74a16f9c4b..e461007558e89 100644 --- a/net/netfilter/nft_dynset.c +++ b/net/netfilter/nft_dynset.c @@ -240,11 +240,15 @@ static void nft_dynset_deactivate(const struct nft_ctx *ctx, { struct nft_dynset *priv = nft_expr_priv(expr); - if (phase == NFT_TRANS_PREPARE) - return; + nf_tables_deactivate_set(ctx, priv->set, &priv->binding, phase); +} + +static void nft_dynset_activate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + struct nft_dynset *priv = nft_expr_priv(expr); - nf_tables_unbind_set(ctx, priv->set, &priv->binding, - phase == NFT_TRANS_COMMIT); + priv->set->use++; } static void nft_dynset_destroy(const struct nft_ctx *ctx, @@ -292,6 +296,7 @@ static const struct nft_expr_ops nft_dynset_ops = { .eval = nft_dynset_eval, .init = nft_dynset_init, .destroy = nft_dynset_destroy, + .activate = nft_dynset_activate, .deactivate = nft_dynset_deactivate, .dump = nft_dynset_dump, }; diff --git a/net/netfilter/nft_lookup.c b/net/netfilter/nft_lookup.c index 14496da5141d3..161c3451a747a 100644 --- a/net/netfilter/nft_lookup.c +++ b/net/netfilter/nft_lookup.c @@ -127,11 +127,15 @@ static void nft_lookup_deactivate(const struct nft_ctx *ctx, { struct nft_lookup *priv = nft_expr_priv(expr); - if (phase == NFT_TRANS_PREPARE) - return; + nf_tables_deactivate_set(ctx, priv->set, &priv->binding, phase); +} + +static void nft_lookup_activate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + struct nft_lookup *priv = nft_expr_priv(expr); - nf_tables_unbind_set(ctx, priv->set, &priv->binding, - phase == NFT_TRANS_COMMIT); + priv->set->use++; } static void nft_lookup_destroy(const struct nft_ctx *ctx, @@ -222,6 +226,7 @@ static const struct nft_expr_ops nft_lookup_ops = { .size = NFT_EXPR_SIZE(sizeof(struct nft_lookup)), .eval = nft_lookup_eval, .init = nft_lookup_init, + .activate = nft_lookup_activate, .deactivate = nft_lookup_deactivate, .destroy = nft_lookup_destroy, .dump = nft_lookup_dump, diff --git a/net/netfilter/nft_objref.c b/net/netfilter/nft_objref.c index 79ef074c18caf..457a9ceb46af2 100644 --- a/net/netfilter/nft_objref.c +++ b/net/netfilter/nft_objref.c @@ -162,11 +162,15 @@ static void nft_objref_map_deactivate(const struct nft_ctx *ctx, { struct nft_objref_map *priv = nft_expr_priv(expr); - if (phase == NFT_TRANS_PREPARE) - return; + nf_tables_deactivate_set(ctx, priv->set, &priv->binding, phase); +} + +static void nft_objref_map_activate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + struct nft_objref_map *priv = nft_expr_priv(expr); - nf_tables_unbind_set(ctx, priv->set, &priv->binding, - phase == NFT_TRANS_COMMIT); + priv->set->use++; } static void nft_objref_map_destroy(const struct nft_ctx *ctx, @@ -183,6 +187,7 @@ static const struct nft_expr_ops nft_objref_map_ops = { .size = NFT_EXPR_SIZE(sizeof(struct nft_objref_map)), .eval = nft_objref_map_eval, .init = nft_objref_map_init, + .activate = nft_objref_map_activate, .deactivate = nft_objref_map_deactivate, .destroy = nft_objref_map_destroy, .dump = nft_objref_map_dump, diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 2c2aac4ac721c..dc10525e90e70 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -1893,6 +1893,7 @@ static int tfilter_notify(struct net *net, struct sk_buff *oskb, { struct sk_buff *skb; u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; + int err = 0; skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); if (!skb) @@ -1906,10 +1907,14 @@ static int tfilter_notify(struct net *net, struct sk_buff *oskb, } if (unicast) - return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); + err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); + else + err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, + n->nlmsg_flags & NLM_F_ECHO); - return rtnetlink_send(skb, net, portid, RTNLGRP_TC, - n->nlmsg_flags & NLM_F_ECHO); + if (err > 0) + err = 0; + return err; } static int tfilter_del_notify(struct net *net, struct sk_buff *oskb, @@ -1941,12 +1946,15 @@ static int tfilter_del_notify(struct net *net, struct sk_buff *oskb, } if (unicast) - return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); - - err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, - n->nlmsg_flags & NLM_F_ECHO); + err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); + else + err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, + n->nlmsg_flags & NLM_F_ECHO); if (err < 0) NL_SET_ERR_MSG(extack, "Failed to send filter delete notification"); + + if (err > 0) + err = 0; return err; } @@ -2688,6 +2696,7 @@ static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb, struct tcf_block *block = chain->block; struct net *net = block->net; struct sk_buff *skb; + int err = 0; skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); if (!skb) @@ -2701,9 +2710,14 @@ static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb, } if (unicast) - return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); + err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); + else + err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, + flags & NLM_F_ECHO); - return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO); + if (err > 0) + err = 0; + return err; } static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops, diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 352b46f98440a..fb8f138b97763 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -1824,6 +1824,7 @@ static int tclass_notify(struct net *net, struct sk_buff *oskb, { struct sk_buff *skb; u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; + int err = 0; skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); if (!skb) @@ -1834,8 +1835,11 @@ static int tclass_notify(struct net *net, struct sk_buff *oskb, return -EINVAL; } - return rtnetlink_send(skb, net, portid, RTNLGRP_TC, - n->nlmsg_flags & NLM_F_ECHO); + err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, + n->nlmsg_flags & NLM_F_ECHO); + if (err > 0) + err = 0; + return err; } static int tclass_del_notify(struct net *net, @@ -1866,8 +1870,11 @@ static int tclass_del_notify(struct net *net, return err; } - return rtnetlink_send(skb, net, portid, RTNLGRP_TC, - n->nlmsg_flags & NLM_F_ECHO); + err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, + n->nlmsg_flags & NLM_F_ECHO); + if (err > 0) + err = 0; + return err; } #ifdef CONFIG_NET_CLS diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 4216fe33204a8..228970e6e52ba 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -1730,7 +1730,12 @@ call_allocate(struct rpc_task *task) req->rq_callsize = RPC_CALLHDRSIZE + (auth->au_cslack << 1) + proc->p_arglen; req->rq_callsize <<= 2; - req->rq_rcvsize = RPC_REPHDRSIZE + auth->au_rslack + proc->p_replen; + /* + * Note: the reply buffer must at minimum allocate enough space + * for the 'struct accepted_reply' from RFC5531. + */ + req->rq_rcvsize = RPC_REPHDRSIZE + auth->au_rslack + \ + max_t(size_t, proc->p_replen, 2); req->rq_rcvsize <<= 2; status = xprt->ops->buf_alloc(task); @@ -2387,9 +2392,6 @@ call_decode(struct rpc_task *task) WARN_ON(memcmp(&req->rq_rcv_buf, &req->rq_private_buf, sizeof(req->rq_rcv_buf)) != 0); - if (req->rq_rcv_buf.len < 12) - goto out_retry; - xdr_init_decode(&xdr, &req->rq_rcv_buf, req->rq_rcv_buf.head[0].iov_base, req); switch (rpc_decode_header(task, &xdr)) { @@ -2400,7 +2402,6 @@ call_decode(struct rpc_task *task) task->tk_pid, __func__, task->tk_status); return; case -EAGAIN: -out_retry: task->tk_status = 0; /* Note: rpc_decode_header() may have freed the RPC slot */ if (task->tk_rqstp == req) { @@ -2449,7 +2450,7 @@ static noinline int rpc_decode_header(struct rpc_task *task, struct xdr_stream *xdr) { struct rpc_clnt *clnt = task->tk_client; - int error = -EACCES; + int error; __be32 *p; /* RFC-1014 says that the representation of XDR data must be a @@ -2458,7 +2459,7 @@ rpc_decode_header(struct rpc_task *task, struct xdr_stream *xdr) * undefined results */ if (task->tk_rqstp->rq_rcv_buf.len & 3) - goto out_badlen; + goto out_unparsable; p = xdr_inline_decode(xdr, 3 * sizeof(*p)); if (!p) @@ -2492,10 +2493,12 @@ rpc_decode_header(struct rpc_task *task, struct xdr_stream *xdr) error = -EOPNOTSUPP; goto out_err; case rpc_garbage_args: + case rpc_system_err: trace_rpc__garbage_args(task); + error = -EIO; break; default: - trace_rpc__unparsable(task); + goto out_unparsable; } out_garbage: @@ -2509,11 +2512,6 @@ out_err: rpc_exit(task, error); return error; -out_badlen: - trace_rpc__unparsable(task); - error = -EIO; - goto out_err; - out_unparsable: trace_rpc__unparsable(task); error = -EIO; @@ -2524,6 +2522,7 @@ out_verifier: goto out_garbage; out_msg_denied: + error = -EACCES; p = xdr_inline_decode(xdr, sizeof(*p)); if (!p) goto out_unparsable; @@ -2535,9 +2534,7 @@ out_msg_denied: error = -EPROTONOSUPPORT; goto out_err; default: - trace_rpc__unparsable(task); - error = -EIO; - goto out_err; + goto out_unparsable; } p = xdr_inline_decode(xdr, sizeof(*p)); @@ -2572,8 +2569,7 @@ out_msg_denied: task->tk_xprt->servername); break; default: - trace_rpc__unparsable(task); - error = -EIO; + goto out_unparsable; } goto out_err; } diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c index e096c5a725dff..d7117d2414601 100644 --- a/net/sunrpc/xprt.c +++ b/net/sunrpc/xprt.c @@ -664,7 +664,7 @@ void xprt_disconnect_done(struct rpc_xprt *xprt) spin_lock_bh(&xprt->transport_lock); xprt_clear_connected(xprt); xprt_clear_write_space_locked(xprt); - xprt_wake_pending_tasks(xprt, -EAGAIN); + xprt_wake_pending_tasks(xprt, -ENOTCONN); spin_unlock_bh(&xprt->transport_lock); } EXPORT_SYMBOL_GPL(xprt_disconnect_done); diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 42f45d33dc567..9359539907baf 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -453,7 +453,7 @@ xs_read_xdr_buf(struct socket *sock, struct msghdr *msg, int flags, goto out; if (ret != want) goto out; - } else + } else if (offset < seek_init) offset = seek_init; ret = -EMSGSIZE; out: diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 4a1da837a733d..135a7ee9db034 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -558,9 +558,6 @@ void tls_device_write_space(struct sock *sk, struct tls_context *ctx) MSG_DONTWAIT | MSG_NOSIGNAL); sk->sk_allocation = sk_allocation; } - - if (!rc) - ctx->sk_write_space(sk); } void handle_device_resync(struct sock *sk, u32 seq, u64 rcd_sn) diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c index 17e8667917aa3..df921a2904b9b 100644 --- a/net/tls/tls_main.c +++ b/net/tls/tls_main.c @@ -146,7 +146,6 @@ retry: } ctx->in_tcp_sendpages = false; - ctx->sk_write_space(sk); return 0; } @@ -228,6 +227,8 @@ static void tls_write_space(struct sock *sk) else #endif tls_sw_write_space(sk, ctx); + + ctx->sk_write_space(sk); } static void tls_ctx_free(struct tls_context *ctx) diff --git a/net/x25/af_x25.c b/net/x25/af_x25.c index 27171ac6fe3b3..20a511398389d 100644 --- a/net/x25/af_x25.c +++ b/net/x25/af_x25.c @@ -825,6 +825,7 @@ out_put_neigh: x25_neigh_put(x25->neighbour); x25->neighbour = NULL; read_unlock_bh(&x25_list_lock); + x25->state = X25_STATE_0; } out_put_route: x25_route_put(rt); diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index 3f80a684c232a..fefee040bf791 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -356,6 +356,7 @@ static void aafs_remove(struct dentry *dentry) simple_rmdir(dir, dentry); else simple_unlink(dir, dentry); + d_delete(dentry); dput(dentry); } inode_unlock(dir); diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index 379682e2a8d5d..f6c2bcb2ab145 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -579,6 +579,7 @@ fail: kfree(profile->secmark[i].label); kfree(profile->secmark); profile->secmark_count = 0; + profile->secmark = NULL; } e->pos = pos; diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index c5363f0c67ef7..1d0b37af2444d 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -939,8 +939,11 @@ static int selinux_sb_clone_mnt_opts(const struct super_block *oldsb, BUG_ON(!(oldsbsec->flags & SE_SBINITIALIZED)); /* if fs is reusing a sb, make sure that the contexts match */ - if (newsbsec->flags & SE_SBINITIALIZED) + if (newsbsec->flags & SE_SBINITIALIZED) { + if ((kern_flags & SECURITY_LSM_NATIVE_LABELS) && !set_context) + *set_kern_flags |= SECURITY_LSM_NATIVE_LABELS; return selinux_cmp_sb_context(oldsb, newsb); + } mutex_lock(&newsbsec->lock); @@ -5134,6 +5137,9 @@ static int selinux_sctp_bind_connect(struct sock *sk, int optname, return -EINVAL; } + if (walk_size + len > addrlen) + return -EINVAL; + err = -EINVAL; switch (optname) { /* Bind checks */ diff --git a/sound/hda/hdac_stream.c b/sound/hda/hdac_stream.c index f5dd288d1a7a3..76e9b41fcea2c 100644 --- a/sound/hda/hdac_stream.c +++ b/sound/hda/hdac_stream.c @@ -95,7 +95,10 @@ void snd_hdac_stream_start(struct hdac_stream *azx_dev, bool fresh_start) 1 << azx_dev->index, 1 << azx_dev->index); /* set stripe control */ - stripe_ctl = snd_hdac_get_stream_stripe_ctl(bus, azx_dev->substream); + if (azx_dev->substream) + stripe_ctl = snd_hdac_get_stream_stripe_ctl(bus, azx_dev->substream); + else + stripe_ctl = 0; snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, stripe_ctl); /* set DMA start and interrupt mask */ diff --git a/sound/pci/hda/hda_tegra.c b/sound/pci/hda/hda_tegra.c index dbd8da5685cb5..3d68f9ef76944 100644 --- a/sound/pci/hda/hda_tegra.c +++ b/sound/pci/hda/hda_tegra.c @@ -219,7 +219,6 @@ disable_hda: return rc; } -#ifdef CONFIG_PM_SLEEP static void hda_tegra_disable_clocks(struct hda_tegra *data) { clk_disable_unprepare(data->hda2hdmi_clk); @@ -230,7 +229,7 @@ static void hda_tegra_disable_clocks(struct hda_tegra *data) /* * power management */ -static int hda_tegra_suspend(struct device *dev) +static int __maybe_unused hda_tegra_suspend(struct device *dev) { struct snd_card *card = dev_get_drvdata(dev); int rc; @@ -243,7 +242,7 @@ static int hda_tegra_suspend(struct device *dev) return 0; } -static int hda_tegra_resume(struct device *dev) +static int __maybe_unused hda_tegra_resume(struct device *dev) { struct snd_card *card = dev_get_drvdata(dev); int rc; @@ -255,10 +254,8 @@ static int hda_tegra_resume(struct device *dev) return 0; } -#endif /* CONFIG_PM_SLEEP */ -#ifdef CONFIG_PM -static int hda_tegra_runtime_suspend(struct device *dev) +static int __maybe_unused hda_tegra_runtime_suspend(struct device *dev) { struct snd_card *card = dev_get_drvdata(dev); struct azx *chip = card->private_data; @@ -275,7 +272,7 @@ static int hda_tegra_runtime_suspend(struct device *dev) return 0; } -static int hda_tegra_runtime_resume(struct device *dev) +static int __maybe_unused hda_tegra_runtime_resume(struct device *dev) { struct snd_card *card = dev_get_drvdata(dev); struct azx *chip = card->private_data; @@ -292,7 +289,6 @@ static int hda_tegra_runtime_resume(struct device *dev) return 0; } -#endif /* CONFIG_PM */ static const struct dev_pm_ops hda_tegra_pm = { SET_SYSTEM_SLEEP_PM_OPS(hda_tegra_suspend, hda_tegra_resume) diff --git a/sound/pci/hda/patch_conexant.c b/sound/pci/hda/patch_conexant.c index a4ee7656d9ee9..fb65ad31e86c8 100644 --- a/sound/pci/hda/patch_conexant.c +++ b/sound/pci/hda/patch_conexant.c @@ -936,6 +936,9 @@ static const struct snd_pci_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x103c, 0x8299, "HP 800 G3 SFF", CXT_FIXUP_HP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x829a, "HP 800 G3 DM", CXT_FIXUP_HP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x8455, "HP Z2 G4", CXT_FIXUP_HP_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x103c, 0x8456, "HP Z2 G4 SFF", CXT_FIXUP_HP_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x103c, 0x8457, "HP Z2 G4 mini", CXT_FIXUP_HP_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x103c, 0x8458, "HP Z2 G4 mini premium", CXT_FIXUP_HP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x138d, "Asus", CXT_FIXUP_HEADPHONE_MIC_PIN), SND_PCI_QUIRK(0x152d, 0x0833, "OLPC XO-1.5", CXT_FIXUP_OLPC_XO), SND_PCI_QUIRK(0x17aa, 0x20f2, "Lenovo T400", CXT_PINCFG_LENOVO_TP410), diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c index 73d7042ff8841..8b3ac690efa36 100644 --- a/sound/pci/hda/patch_hdmi.c +++ b/sound/pci/hda/patch_hdmi.c @@ -57,10 +57,11 @@ MODULE_PARM_DESC(static_hdmi_pcm, "Don't restrict PCM parameters per ELD info"); #define is_geminilake(codec) (((codec)->core.vendor_id == 0x8086280d) || \ ((codec)->core.vendor_id == 0x80862800)) #define is_cannonlake(codec) ((codec)->core.vendor_id == 0x8086280c) +#define is_icelake(codec) ((codec)->core.vendor_id == 0x8086280f) #define is_haswell_plus(codec) (is_haswell(codec) || is_broadwell(codec) \ || is_skylake(codec) || is_broxton(codec) \ - || is_kabylake(codec)) || is_geminilake(codec) \ - || is_cannonlake(codec) + || is_kabylake(codec) || is_geminilake(codec) \ + || is_cannonlake(codec) || is_icelake(codec)) #define is_valleyview(codec) ((codec)->core.vendor_id == 0x80862882) #define is_cherryview(codec) ((codec)->core.vendor_id == 0x80862883) #define is_valleyview_plus(codec) (is_valleyview(codec) || is_cherryview(codec)) @@ -181,6 +182,8 @@ struct hdmi_spec { struct hdac_chmap chmap; hda_nid_t vendor_nid; + const int *port_map; + int port_num; }; #ifdef CONFIG_SND_HDA_COMPONENT @@ -2418,12 +2421,11 @@ static void intel_haswell_fixup_connect_list(struct hda_codec *codec, snd_hda_override_conn_list(codec, nid, spec->num_cvts, spec->cvt_nids); } -#define INTEL_VENDOR_NID 0x08 -#define INTEL_GLK_VENDOR_NID 0x0B -#define INTEL_GET_VENDOR_VERB 0xf81 -#define INTEL_SET_VENDOR_VERB 0x781 -#define INTEL_EN_DP12 0x02 /* enable DP 1.2 features */ -#define INTEL_EN_ALL_PIN_CVTS 0x01 /* enable 2nd & 3rd pins and convertors */ +#define INTEL_GET_VENDOR_VERB 0xf81 +#define INTEL_GET_VENDOR_VERB 0xf81 +#define INTEL_SET_VENDOR_VERB 0x781 +#define INTEL_EN_DP12 0x02 /* enable DP 1.2 features */ +#define INTEL_EN_ALL_PIN_CVTS 0x01 /* enable 2nd & 3rd pins and convertors */ static void intel_haswell_enable_all_pins(struct hda_codec *codec, bool update_tree) @@ -2503,11 +2505,29 @@ static int intel_base_nid(struct hda_codec *codec) static int intel_pin2port(void *audio_ptr, int pin_nid) { - int base_nid = intel_base_nid(audio_ptr); + struct hda_codec *codec = audio_ptr; + struct hdmi_spec *spec = codec->spec; + int base_nid, i; - if (WARN_ON(pin_nid < base_nid || pin_nid >= base_nid + 3)) - return -1; - return pin_nid - base_nid + 1; /* intel port is 1-based */ + if (!spec->port_num) { + base_nid = intel_base_nid(codec); + if (WARN_ON(pin_nid < base_nid || pin_nid >= base_nid + 3)) + return -1; + return pin_nid - base_nid + 1; /* intel port is 1-based */ + } + + /* + * looking for the pin number in the mapping table and return + * the index which indicate the port number + */ + for (i = 0; i < spec->port_num; i++) { + if (pin_nid == spec->port_map[i]) + return i + 1; + } + + /* return -1 if pin number exceeds our expectation */ + codec_info(codec, "Can't find the HDMI/DP port for pin %d\n", pin_nid); + return -1; } static void intel_pin_eld_notify(void *audio_ptr, int port, int pipe) @@ -2608,7 +2628,8 @@ static int parse_intel_hdmi(struct hda_codec *codec) } /* Intel Haswell and onwards; audio component with eld notifier */ -static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid) +static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid, + const int *port_map, int port_num) { struct hdmi_spec *spec; int err; @@ -2620,6 +2641,8 @@ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid) codec->dp_mst = true; spec->dyn_pcm_assign = true; spec->vendor_nid = vendor_nid; + spec->port_map = port_map; + spec->port_num = port_num; intel_haswell_enable_all_pins(codec, true); intel_haswell_fixup_enable_dp12(codec); @@ -2638,12 +2661,23 @@ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid) static int patch_i915_hsw_hdmi(struct hda_codec *codec) { - return intel_hsw_common_init(codec, INTEL_VENDOR_NID); + return intel_hsw_common_init(codec, 0x08, NULL, 0); } static int patch_i915_glk_hdmi(struct hda_codec *codec) { - return intel_hsw_common_init(codec, INTEL_GLK_VENDOR_NID); + return intel_hsw_common_init(codec, 0x0b, NULL, 0); +} + +static int patch_i915_icl_hdmi(struct hda_codec *codec) +{ + /* + * pin to port mapping table where the value indicate the pin number and + * the index indicate the port number with 1 base. + */ + static const int map[] = {0x4, 0x6, 0x8, 0xa, 0xb}; + + return intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map)); } /* Intel Baytrail and Braswell; with eld notifier */ @@ -3886,6 +3920,7 @@ HDA_CODEC_ENTRY(0x11069f81, "VX900 HDMI/DP", patch_via_hdmi), HDA_CODEC_ENTRY(0x11069f84, "VX11 HDMI/DP", patch_generic_hdmi), HDA_CODEC_ENTRY(0x11069f85, "VX11 HDMI/DP", patch_generic_hdmi), HDA_CODEC_ENTRY(0x80860054, "IbexPeak HDMI", patch_i915_cpt_hdmi), +HDA_CODEC_ENTRY(0x80862800, "Geminilake HDMI", patch_i915_glk_hdmi), HDA_CODEC_ENTRY(0x80862801, "Bearlake HDMI", patch_generic_hdmi), HDA_CODEC_ENTRY(0x80862802, "Cantiga HDMI", patch_generic_hdmi), HDA_CODEC_ENTRY(0x80862803, "Eaglelake HDMI", patch_generic_hdmi), @@ -3899,7 +3934,7 @@ HDA_CODEC_ENTRY(0x8086280a, "Broxton HDMI", patch_i915_hsw_hdmi), HDA_CODEC_ENTRY(0x8086280b, "Kabylake HDMI", patch_i915_hsw_hdmi), HDA_CODEC_ENTRY(0x8086280c, "Cannonlake HDMI", patch_i915_glk_hdmi), HDA_CODEC_ENTRY(0x8086280d, "Geminilake HDMI", patch_i915_glk_hdmi), -HDA_CODEC_ENTRY(0x80862800, "Geminilake HDMI", patch_i915_glk_hdmi), +HDA_CODEC_ENTRY(0x8086280f, "Icelake HDMI", patch_i915_icl_hdmi), HDA_CODEC_ENTRY(0x80862880, "CedarTrail HDMI", patch_generic_hdmi), HDA_CODEC_ENTRY(0x80862882, "Valleyview2 HDMI", patch_i915_byt_hdmi), HDA_CODEC_ENTRY(0x80862883, "Braswell HDMI", patch_i915_byt_hdmi), diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index c8413d44973ca..384719d5c44ec 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -5491,7 +5491,7 @@ static void alc_headset_btn_callback(struct hda_codec *codec, jack->jack->button_state = report; } -static void alc_fixup_headset_jack(struct hda_codec *codec, +static void alc295_fixup_chromebook(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -5501,6 +5501,16 @@ static void alc_fixup_headset_jack(struct hda_codec *codec, alc_headset_btn_callback); snd_hda_jack_add_kctl(codec, 0x55, "Headset Jack", false, SND_JACK_HEADSET, alc_headset_btn_keymap); + switch (codec->core.vendor_id) { + case 0x10ec0295: + alc_update_coef_idx(codec, 0x4a, 0x8000, 1 << 15); /* Reset HP JD */ + alc_update_coef_idx(codec, 0x4a, 0x8000, 0 << 15); + break; + case 0x10ec0236: + alc_update_coef_idx(codec, 0x1b, 0x8000, 1 << 15); /* Reset HP JD */ + alc_update_coef_idx(codec, 0x1b, 0x8000, 0 << 15); + break; + } break; case HDA_FIXUP_ACT_INIT: switch (codec->core.vendor_id) { @@ -5670,9 +5680,13 @@ enum { ALC294_FIXUP_ASUS_MIC, ALC294_FIXUP_ASUS_HEADSET_MIC, ALC294_FIXUP_ASUS_SPK, - ALC225_FIXUP_HEADSET_JACK, ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE, ALC285_FIXUP_LENOVO_PC_BEEP_IN_NOISE, + ALC255_FIXUP_ACER_HEADSET_MIC, + ALC295_FIXUP_CHROME_BOOK, + ALC225_FIXUP_DELL_WYSE_AIO_MIC_NO_PRESENCE, + ALC225_FIXUP_WYSE_AUTO_MUTE, + ALC225_FIXUP_WYSE_DISABLE_MIC_VREF, }; static const struct hda_fixup alc269_fixups[] = { @@ -6615,9 +6629,9 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC294_FIXUP_ASUS_HEADSET_MIC }, - [ALC225_FIXUP_HEADSET_JACK] = { + [ALC295_FIXUP_CHROME_BOOK] = { .type = HDA_FIXUP_FUNC, - .v.func = alc_fixup_headset_jack, + .v.func = alc295_fixup_chromebook, }, [ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE] = { .type = HDA_FIXUP_PINS, @@ -6639,6 +6653,38 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC285_FIXUP_LENOVO_HEADPHONE_NOISE }, + [ALC255_FIXUP_ACER_HEADSET_MIC] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x19, 0x03a11130 }, + { 0x1a, 0x90a60140 }, /* use as internal mic */ + { } + }, + .chained = true, + .chain_id = ALC255_FIXUP_HEADSET_MODE_NO_HP_MIC + }, + [ALC225_FIXUP_DELL_WYSE_AIO_MIC_NO_PRESENCE] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x16, 0x01011020 }, /* Rear Line out */ + { 0x19, 0x01a1913c }, /* use as Front headset mic, without its own jack detect */ + { } + }, + .chained = true, + .chain_id = ALC225_FIXUP_WYSE_AUTO_MUTE + }, + [ALC225_FIXUP_WYSE_AUTO_MUTE] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc_fixup_auto_mute_via_amp, + .chained = true, + .chain_id = ALC225_FIXUP_WYSE_DISABLE_MIC_VREF + }, + [ALC225_FIXUP_WYSE_DISABLE_MIC_VREF] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc_fixup_disable_mic_vref, + .chained = true, + .chain_id = ALC269_FIXUP_HEADSET_MODE_NO_HP_MIC + }, }; static const struct snd_pci_quirk alc269_fixup_tbl[] = { @@ -6658,6 +6704,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x128f, "Acer Veriton Z6860G", ALC286_FIXUP_ACER_AIO_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1025, 0x1290, "Acer Veriton Z4860G", ALC286_FIXUP_ACER_AIO_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1025, 0x1291, "Acer Veriton Z4660G", ALC286_FIXUP_ACER_AIO_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x1025, 0x1330, "Acer TravelMate X514-51T", ALC255_FIXUP_ACER_HEADSET_MIC), SND_PCI_QUIRK(0x1028, 0x0470, "Dell M101z", ALC269_FIXUP_DELL_M101Z), SND_PCI_QUIRK(0x1028, 0x054b, "Dell XPS one 2710", ALC275_FIXUP_DELL_XPS), SND_PCI_QUIRK(0x1028, 0x05bd, "Dell Latitude E6440", ALC292_FIXUP_DELL_E7X), @@ -6702,6 +6749,8 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1028, 0x0871, "Dell Precision 3630", ALC255_FIXUP_DELL_HEADSET_MIC), SND_PCI_QUIRK(0x1028, 0x0872, "Dell Precision 3630", ALC255_FIXUP_DELL_HEADSET_MIC), SND_PCI_QUIRK(0x1028, 0x0873, "Dell Precision 3930", ALC255_FIXUP_DUMMY_LINEOUT_VERB), + SND_PCI_QUIRK(0x1028, 0x08ad, "Dell WYSE AIO", ALC225_FIXUP_DELL_WYSE_AIO_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x1028, 0x08ae, "Dell WYSE NB", ALC225_FIXUP_DELL1_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x0935, "Dell", ALC274_FIXUP_DELL_AIO_LINEOUT_VERB), SND_PCI_QUIRK(0x1028, 0x164a, "Dell", ALC293_FIXUP_DELL1_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x164b, "Dell", ALC293_FIXUP_DELL1_MIC_NO_PRESENCE), @@ -6764,11 +6813,13 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x2336, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x2337, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x221c, "HP EliteBook 755 G2", ALC280_FIXUP_HP_HEADSET_MIC), + SND_PCI_QUIRK(0x103c, 0x802e, "HP Z240 SFF", ALC221_FIXUP_HP_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x103c, 0x802f, "HP Z240", ALC221_FIXUP_HP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x820d, "HP Pavilion 15", ALC269_FIXUP_HP_MUTE_LED_MIC3), SND_PCI_QUIRK(0x103c, 0x8256, "HP", ALC221_FIXUP_HP_FRONT_MIC), SND_PCI_QUIRK(0x103c, 0x827e, "HP x360", ALC295_FIXUP_HP_X360), - SND_PCI_QUIRK(0x103c, 0x82bf, "HP", ALC221_FIXUP_HP_MIC_NO_PRESENCE), - SND_PCI_QUIRK(0x103c, 0x82c0, "HP", ALC221_FIXUP_HP_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x103c, 0x82bf, "HP G3 mini", ALC221_FIXUP_HP_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x103c, 0x82c0, "HP G3 mini premium", ALC221_FIXUP_HP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x83b9, "HP Spectre x360", ALC269_FIXUP_HP_MUTE_LED_MIC3), SND_PCI_QUIRK(0x1043, 0x103e, "ASUS X540SA", ALC256_FIXUP_ASUS_MIC), SND_PCI_QUIRK(0x1043, 0x103f, "ASUS TX300", ALC282_FIXUP_ASUS_TX300), @@ -7048,7 +7099,7 @@ static const struct hda_model_fixup alc269_fixup_models[] = { {.id = ALC255_FIXUP_DUMMY_LINEOUT_VERB, .name = "alc255-dummy-lineout"}, {.id = ALC255_FIXUP_DELL_HEADSET_MIC, .name = "alc255-dell-headset"}, {.id = ALC295_FIXUP_HP_X360, .name = "alc295-hp-x360"}, - {.id = ALC225_FIXUP_HEADSET_JACK, .name = "alc-sense-combo"}, + {.id = ALC295_FIXUP_CHROME_BOOK, .name = "alc-sense-combo"}, {} }; #define ALC225_STANDARD_PINS \ diff --git a/sound/usb/usx2y/usb_stream.c b/sound/usb/usx2y/usb_stream.c index b0f8979ff2d2f..221adf68bd0cb 100644 --- a/sound/usb/usx2y/usb_stream.c +++ b/sound/usb/usx2y/usb_stream.c @@ -104,7 +104,12 @@ static int init_urbs(struct usb_stream_kernel *sk, unsigned use_packsize, for (u = 0; u < USB_STREAM_NURBS; ++u) { sk->inurb[u] = usb_alloc_urb(sk->n_o_ps, GFP_KERNEL); + if (!sk->inurb[u]) + return -ENOMEM; + sk->outurb[u] = usb_alloc_urb(sk->n_o_ps, GFP_KERNEL); + if (!sk->outurb[u]) + return -ENOMEM; } if (init_pipe_urbs(sk, use_packsize, sk->inurb, indata, dev, in_pipe) || diff --git a/tools/power/cpupower/lib/cpufreq.c b/tools/power/cpupower/lib/cpufreq.c index 0c0f3e3f0d803..80650497fb800 100644 --- a/tools/power/cpupower/lib/cpufreq.c +++ b/tools/power/cpupower/lib/cpufreq.c @@ -333,17 +333,20 @@ void cpufreq_put_available_governors(struct cpufreq_available_governors *any) } -struct cpufreq_available_frequencies -*cpufreq_get_available_frequencies(unsigned int cpu) +struct cpufreq_frequencies +*cpufreq_get_frequencies(const char *type, unsigned int cpu) { - struct cpufreq_available_frequencies *first = NULL; - struct cpufreq_available_frequencies *current = NULL; + struct cpufreq_frequencies *first = NULL; + struct cpufreq_frequencies *current = NULL; char one_value[SYSFS_PATH_MAX]; char linebuf[MAX_LINE_LEN]; + char fname[MAX_LINE_LEN]; unsigned int pos, i; unsigned int len; - len = sysfs_cpufreq_read_file(cpu, "scaling_available_frequencies", + snprintf(fname, MAX_LINE_LEN, "scaling_%s_frequencies", type); + + len = sysfs_cpufreq_read_file(cpu, fname, linebuf, sizeof(linebuf)); if (len == 0) return NULL; @@ -389,9 +392,9 @@ struct cpufreq_available_frequencies return NULL; } -void cpufreq_put_available_frequencies(struct cpufreq_available_frequencies - *any) { - struct cpufreq_available_frequencies *tmp, *next; +void cpufreq_put_frequencies(struct cpufreq_frequencies *any) +{ + struct cpufreq_frequencies *tmp, *next; if (!any) return; diff --git a/tools/power/cpupower/lib/cpufreq.h b/tools/power/cpupower/lib/cpufreq.h index 60beaf5ed2ea4..775738269cbfc 100644 --- a/tools/power/cpupower/lib/cpufreq.h +++ b/tools/power/cpupower/lib/cpufreq.h @@ -28,10 +28,10 @@ struct cpufreq_available_governors { struct cpufreq_available_governors *first; }; -struct cpufreq_available_frequencies { +struct cpufreq_frequencies { unsigned long frequency; - struct cpufreq_available_frequencies *next; - struct cpufreq_available_frequencies *first; + struct cpufreq_frequencies *next; + struct cpufreq_frequencies *first; }; @@ -129,14 +129,14 @@ void cpufreq_put_available_governors( * * Only present on _some_ ->target() cpufreq drivers. For information purposes * only. Please free allocated memory by calling - * cpufreq_put_available_frequencies after use. + * cpufreq_put_frequencies after use. */ -struct cpufreq_available_frequencies -*cpufreq_get_available_frequencies(unsigned int cpu); +struct cpufreq_frequencies +*cpufreq_get_frequencies(const char *type, unsigned int cpu); -void cpufreq_put_available_frequencies( - struct cpufreq_available_frequencies *first); +void cpufreq_put_frequencies( + struct cpufreq_frequencies *first); /* determine affected CPUs diff --git a/tools/power/cpupower/utils/cpufreq-info.c b/tools/power/cpupower/utils/cpufreq-info.c index c3f39d5128ee4..10290b308797a 100644 --- a/tools/power/cpupower/utils/cpufreq-info.c +++ b/tools/power/cpupower/utils/cpufreq-info.c @@ -161,19 +161,12 @@ static void print_duration(unsigned long duration) return; } -/* --boost / -b */ - -static int get_boost_mode(unsigned int cpu) +static int get_boost_mode_x86(unsigned int cpu) { int support, active, b_states = 0, ret, pstate_no, i; /* ToDo: Make this more global */ unsigned long pstates[MAX_HW_PSTATES] = {0,}; - if (cpupower_cpu_info.vendor != X86_VENDOR_AMD && - cpupower_cpu_info.vendor != X86_VENDOR_HYGON && - cpupower_cpu_info.vendor != X86_VENDOR_INTEL) - return 0; - ret = cpufreq_has_boost_support(cpu, &support, &active, &b_states); if (ret) { printf(_("Error while evaluating Boost Capabilities" @@ -248,6 +241,33 @@ static int get_boost_mode(unsigned int cpu) return 0; } +/* --boost / -b */ + +static int get_boost_mode(unsigned int cpu) +{ + struct cpufreq_frequencies *freqs; + + if (cpupower_cpu_info.vendor == X86_VENDOR_AMD || + cpupower_cpu_info.vendor == X86_VENDOR_HYGON || + cpupower_cpu_info.vendor == X86_VENDOR_INTEL) + return get_boost_mode_x86(cpu); + + freqs = cpufreq_get_frequencies("boost", cpu); + if (freqs) { + printf(_(" boost frequency steps: ")); + while (freqs->next) { + print_speed(freqs->frequency); + printf(", "); + freqs = freqs->next; + } + print_speed(freqs->frequency); + printf("\n"); + cpufreq_put_frequencies(freqs); + } + + return 0; +} + /* --freq / -f */ static int get_freq_kernel(unsigned int cpu, unsigned int human) @@ -456,7 +476,7 @@ static int get_latency(unsigned int cpu, unsigned int human) static void debug_output_one(unsigned int cpu) { - struct cpufreq_available_frequencies *freqs; + struct cpufreq_frequencies *freqs; get_driver(cpu); get_related_cpus(cpu); @@ -464,7 +484,7 @@ static void debug_output_one(unsigned int cpu) get_latency(cpu, 1); get_hardware_limits(cpu, 1); - freqs = cpufreq_get_available_frequencies(cpu); + freqs = cpufreq_get_frequencies("available", cpu); if (freqs) { printf(_(" available frequency steps: ")); while (freqs->next) { @@ -474,7 +494,7 @@ static void debug_output_one(unsigned int cpu) } print_speed(freqs->frequency); printf("\n"); - cpufreq_put_available_frequencies(freqs); + cpufreq_put_frequencies(freqs); } get_available_governors(cpu); diff --git a/tools/testing/nvdimm/Kbuild b/tools/testing/nvdimm/Kbuild index 10ddf223055ba..e1286d2cdfbf9 100644 --- a/tools/testing/nvdimm/Kbuild +++ b/tools/testing/nvdimm/Kbuild @@ -35,6 +35,8 @@ obj-$(CONFIG_DAX) += dax.o endif obj-$(CONFIG_DEV_DAX) += device_dax.o obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o +obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem_core.o +obj-$(CONFIG_DEV_DAX_PMEM_COMPAT) += dax_pmem_compat.o nfit-y := $(ACPI_SRC)/core.o nfit-y += $(ACPI_SRC)/intel.o @@ -57,6 +59,7 @@ nd_e820-y := $(NVDIMM_SRC)/e820.o nd_e820-y += config_check.o dax-y := $(DAX_SRC)/super.o +dax-y += $(DAX_SRC)/bus.o dax-y += config_check.o device_dax-y := $(DAX_SRC)/device.o @@ -64,7 +67,9 @@ device_dax-y += dax-dev.o device_dax-y += device_dax_test.o device_dax-y += config_check.o -dax_pmem-y := $(DAX_SRC)/pmem.o +dax_pmem-y := $(DAX_SRC)/pmem/pmem.o +dax_pmem_core-y := $(DAX_SRC)/pmem/core.o +dax_pmem_compat-y := $(DAX_SRC)/pmem/compat.o dax_pmem-y += config_check.o libnvdimm-y := $(NVDIMM_SRC)/core.o diff --git a/tools/testing/nvdimm/dax-dev.c b/tools/testing/nvdimm/dax-dev.c index 36ee3d8797c3b..f36e708265b85 100644 --- a/tools/testing/nvdimm/dax-dev.c +++ b/tools/testing/nvdimm/dax-dev.c @@ -17,20 +17,11 @@ phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, unsigned long size) { - struct resource *res; + struct resource *res = &dev_dax->region->res; phys_addr_t addr; - int i; - for (i = 0; i < dev_dax->num_resources; i++) { - res = &dev_dax->res[i]; - addr = pgoff * PAGE_SIZE + res->start; - if (addr >= res->start && addr <= res->end) - break; - pgoff -= PHYS_PFN(resource_size(res)); - } - - if (i < dev_dax->num_resources) { - res = &dev_dax->res[i]; + addr = pgoff * PAGE_SIZE + res->start; + if (addr >= res->start && addr <= res->end) { if (addr + size - 1 <= res->end) { if (get_nfit_res(addr)) { struct page *page; @@ -44,6 +35,5 @@ phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, return addr; } } - return -1; } diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index fb5758ac469e6..971fc84281179 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -32,6 +32,7 @@ TARGETS += net TARGETS += netfilter TARGETS += networking/timestamping TARGETS += nsfs +TARGETS += pidfd TARGETS += powerpc TARGETS += proc TARGETS += pstore diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 6210ba41c29e9..2689d1ea6d7aa 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -3,6 +3,7 @@ /x86_64/platform_info_test /x86_64/set_sregs_test /x86_64/sync_regs_test +/x86_64/vmx_close_while_nested_test /x86_64/vmx_tsc_adjust_test /x86_64/state_test /dirty_log_test diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index f9a0e99384805..3c1f4bdf90000 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -16,6 +16,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/cr4_cpuid_sync_test TEST_GEN_PROGS_x86_64 += x86_64/state_test TEST_GEN_PROGS_x86_64 += x86_64/evmcs_test TEST_GEN_PROGS_x86_64 += x86_64/hyperv_cpuid +TEST_GEN_PROGS_x86_64 += x86_64/vmx_close_while_nested_test TEST_GEN_PROGS_x86_64 += dirty_log_test TEST_GEN_PROGS_x86_64 += clear_dirty_log_test diff --git a/tools/testing/selftests/kvm/x86_64/vmx_close_while_nested_test.c b/tools/testing/selftests/kvm/x86_64/vmx_close_while_nested_test.c new file mode 100644 index 0000000000000..6edec6fd790b7 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/vmx_close_while_nested_test.c @@ -0,0 +1,95 @@ +/* + * vmx_close_while_nested + * + * Copyright (C) 2019, Red Hat, Inc. + * + * This work is licensed under the terms of the GNU GPL, version 2. + * + * Verify that nothing bad happens if a KVM user exits with open + * file descriptors while executing a nested guest. + */ + +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" +#include "vmx.h" + +#include <string.h> +#include <sys/ioctl.h> + +#include "kselftest.h" + +#define VCPU_ID 5 + +enum { + PORT_L0_EXIT = 0x2000, +}; + +/* The virtual machine object. */ +static struct kvm_vm *vm; + +static void l2_guest_code(void) +{ + /* Exit to L0 */ + asm volatile("inb %%dx, %%al" + : : [port] "d" (PORT_L0_EXIT) : "rax"); +} + +static void l1_guest_code(struct vmx_pages *vmx_pages) +{ +#define L2_GUEST_STACK_SIZE 64 + unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; + uint32_t control; + uintptr_t save_cr3; + + GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + GUEST_ASSERT(load_vmcs(vmx_pages)); + + /* Prepare the VMCS for L2 execution. */ + prepare_vmcs(vmx_pages, l2_guest_code, + &l2_guest_stack[L2_GUEST_STACK_SIZE]); + + GUEST_ASSERT(!vmlaunch()); + GUEST_ASSERT(0); +} + +int main(int argc, char *argv[]) +{ + struct vmx_pages *vmx_pages; + vm_vaddr_t vmx_pages_gva; + struct kvm_cpuid_entry2 *entry = kvm_get_supported_cpuid_entry(1); + + if (!(entry->ecx & CPUID_VMX)) { + fprintf(stderr, "nested VMX not enabled, skipping test\n"); + exit(KSFT_SKIP); + } + + vm = vm_create_default(VCPU_ID, 0, (void *) l1_guest_code); + vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid()); + + /* Allocate VMX pages and shared descriptors (vmx_pages). */ + vmx_pages = vcpu_alloc_vmx(vm, &vmx_pages_gva); + vcpu_args_set(vm, VCPU_ID, 1, vmx_pages_gva); + + for (;;) { + volatile struct kvm_run *run = vcpu_state(vm, VCPU_ID); + struct ucall uc; + + vcpu_run(vm, VCPU_ID); + TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, + "Got exit_reason other than KVM_EXIT_IO: %u (%s)\n", + run->exit_reason, + exit_reason_str(run->exit_reason)); + + if (run->io.port == PORT_L0_EXIT) + break; + + switch (get_ucall(vm, VCPU_ID, &uc)) { + case UCALL_ABORT: + TEST_ASSERT(false, "%s", (const char *)uc.args[0]); + /* NOT REACHED */ + default: + TEST_ASSERT(false, "Unknown ucall 0x%x.", uc.cmd); + } + } +} diff --git a/tools/testing/selftests/pidfd/Makefile b/tools/testing/selftests/pidfd/Makefile new file mode 100644 index 0000000000000..deaf8073bc066 --- /dev/null +++ b/tools/testing/selftests/pidfd/Makefile @@ -0,0 +1,6 @@ +CFLAGS += -g -I../../../../usr/include/ + +TEST_GEN_PROGS := pidfd_test + +include ../lib.mk + diff --git a/tools/testing/selftests/pidfd/pidfd_test.c b/tools/testing/selftests/pidfd/pidfd_test.c new file mode 100644 index 0000000000000..d59378a93782e --- /dev/null +++ b/tools/testing/selftests/pidfd/pidfd_test.c @@ -0,0 +1,381 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#define _GNU_SOURCE +#include <errno.h> +#include <fcntl.h> +#include <linux/types.h> +#include <sched.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <syscall.h> +#include <sys/mount.h> +#include <sys/wait.h> +#include <unistd.h> + +#include "../kselftest.h" + +static inline int sys_pidfd_send_signal(int pidfd, int sig, siginfo_t *info, + unsigned int flags) +{ + return syscall(__NR_pidfd_send_signal, pidfd, sig, info, flags); +} + +static int signal_received; + +static void set_signal_received_on_sigusr1(int sig) +{ + if (sig == SIGUSR1) + signal_received = 1; +} + +/* + * Straightforward test to see whether pidfd_send_signal() works is to send + * a signal to ourself. + */ +static int test_pidfd_send_signal_simple_success(void) +{ + int pidfd, ret; + const char *test_name = "pidfd_send_signal send SIGUSR1"; + + pidfd = open("/proc/self", O_DIRECTORY | O_CLOEXEC); + if (pidfd < 0) + ksft_exit_fail_msg( + "%s test: Failed to open process file descriptor\n", + test_name); + + signal(SIGUSR1, set_signal_received_on_sigusr1); + + ret = sys_pidfd_send_signal(pidfd, SIGUSR1, NULL, 0); + close(pidfd); + if (ret < 0) + ksft_exit_fail_msg("%s test: Failed to send signal\n", + test_name); + + if (signal_received != 1) + ksft_exit_fail_msg("%s test: Failed to receive signal\n", + test_name); + + signal_received = 0; + ksft_test_result_pass("%s test: Sent signal\n", test_name); + return 0; +} + +static int wait_for_pid(pid_t pid) +{ + int status, ret; + +again: + ret = waitpid(pid, &status, 0); + if (ret == -1) { + if (errno == EINTR) + goto again; + + return -1; + } + + if (ret != pid) + goto again; + + if (!WIFEXITED(status)) + return -1; + + return WEXITSTATUS(status); +} + +static int test_pidfd_send_signal_exited_fail(void) +{ + int pidfd, ret, saved_errno; + char buf[256]; + pid_t pid; + const char *test_name = "pidfd_send_signal signal exited process"; + + pid = fork(); + if (pid < 0) + ksft_exit_fail_msg("%s test: Failed to create new process\n", + test_name); + + if (pid == 0) + _exit(EXIT_SUCCESS); + + snprintf(buf, sizeof(buf), "/proc/%d", pid); + + pidfd = open(buf, O_DIRECTORY | O_CLOEXEC); + + (void)wait_for_pid(pid); + + if (pidfd < 0) + ksft_exit_fail_msg( + "%s test: Failed to open process file descriptor\n", + test_name); + + ret = sys_pidfd_send_signal(pidfd, 0, NULL, 0); + saved_errno = errno; + close(pidfd); + if (ret == 0) + ksft_exit_fail_msg( + "%s test: Managed to send signal to process even though it should have failed\n", + test_name); + + if (saved_errno != ESRCH) + ksft_exit_fail_msg( + "%s test: Expected to receive ESRCH as errno value but received %d instead\n", + test_name, saved_errno); + + ksft_test_result_pass("%s test: Failed to send signal as expected\n", + test_name); + return 0; +} + +/* + * The kernel reserves 300 pids via RESERVED_PIDS in kernel/pid.c + * That means, when it wraps around any pid < 300 will be skipped. + * So we need to use a pid > 300 in order to test recycling. + */ +#define PID_RECYCLE 1000 + +/* + * Maximum number of cycles we allow. This is equivalent to PID_MAX_DEFAULT. + * If users set a higher limit or we have cycled PIDFD_MAX_DEFAULT number of + * times then we skip the test to not go into an infinite loop or block for a + * long time. + */ +#define PIDFD_MAX_DEFAULT 0x8000 + +/* + * Define a few custom error codes for the child process to clearly indicate + * what is happening. This way we can tell the difference between a system + * error, a test error, etc. + */ +#define PIDFD_PASS 0 +#define PIDFD_FAIL 1 +#define PIDFD_ERROR 2 +#define PIDFD_SKIP 3 +#define PIDFD_XFAIL 4 + +static int test_pidfd_send_signal_recycled_pid_fail(void) +{ + int i, ret; + pid_t pid1; + const char *test_name = "pidfd_send_signal signal recycled pid"; + + ret = unshare(CLONE_NEWPID); + if (ret < 0) + ksft_exit_fail_msg("%s test: Failed to unshare pid namespace\n", + test_name); + + ret = unshare(CLONE_NEWNS); + if (ret < 0) + ksft_exit_fail_msg( + "%s test: Failed to unshare mount namespace\n", + test_name); + + ret = mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, 0); + if (ret < 0) + ksft_exit_fail_msg("%s test: Failed to remount / private\n", + test_name); + + /* pid 1 in new pid namespace */ + pid1 = fork(); + if (pid1 < 0) + ksft_exit_fail_msg("%s test: Failed to create new process\n", + test_name); + + if (pid1 == 0) { + char buf[256]; + pid_t pid2; + int pidfd = -1; + + (void)umount2("/proc", MNT_DETACH); + ret = mount("proc", "/proc", "proc", 0, NULL); + if (ret < 0) + _exit(PIDFD_ERROR); + + /* grab pid PID_RECYCLE */ + for (i = 0; i <= PIDFD_MAX_DEFAULT; i++) { + pid2 = fork(); + if (pid2 < 0) + _exit(PIDFD_ERROR); + + if (pid2 == 0) + _exit(PIDFD_PASS); + + if (pid2 == PID_RECYCLE) { + snprintf(buf, sizeof(buf), "/proc/%d", pid2); + ksft_print_msg("pid to recycle is %d\n", pid2); + pidfd = open(buf, O_DIRECTORY | O_CLOEXEC); + } + + if (wait_for_pid(pid2)) + _exit(PIDFD_ERROR); + + if (pid2 >= PID_RECYCLE) + break; + } + + /* + * We want to be as predictable as we can so if we haven't been + * able to grab pid PID_RECYCLE skip the test. + */ + if (pid2 != PID_RECYCLE) { + /* skip test */ + close(pidfd); + _exit(PIDFD_SKIP); + } + + if (pidfd < 0) + _exit(PIDFD_ERROR); + + for (i = 0; i <= PIDFD_MAX_DEFAULT; i++) { + char c; + int pipe_fds[2]; + pid_t recycled_pid; + int child_ret = PIDFD_PASS; + + ret = pipe2(pipe_fds, O_CLOEXEC); + if (ret < 0) + _exit(PIDFD_ERROR); + + recycled_pid = fork(); + if (recycled_pid < 0) + _exit(PIDFD_ERROR); + + if (recycled_pid == 0) { + close(pipe_fds[1]); + (void)read(pipe_fds[0], &c, 1); + close(pipe_fds[0]); + + _exit(PIDFD_PASS); + } + + /* + * Stop the child so we can inspect whether we have + * recycled pid PID_RECYCLE. + */ + close(pipe_fds[0]); + ret = kill(recycled_pid, SIGSTOP); + close(pipe_fds[1]); + if (ret) { + (void)wait_for_pid(recycled_pid); + _exit(PIDFD_ERROR); + } + + /* + * We have recycled the pid. Try to signal it. This + * needs to fail since this is a different process than + * the one the pidfd refers to. + */ + if (recycled_pid == PID_RECYCLE) { + ret = sys_pidfd_send_signal(pidfd, SIGCONT, + NULL, 0); + if (ret && errno == ESRCH) + child_ret = PIDFD_XFAIL; + else + child_ret = PIDFD_FAIL; + } + + /* let the process move on */ + ret = kill(recycled_pid, SIGCONT); + if (ret) + (void)kill(recycled_pid, SIGKILL); + + if (wait_for_pid(recycled_pid)) + _exit(PIDFD_ERROR); + + switch (child_ret) { + case PIDFD_FAIL: + /* fallthrough */ + case PIDFD_XFAIL: + _exit(child_ret); + case PIDFD_PASS: + break; + default: + /* not reached */ + _exit(PIDFD_ERROR); + } + + /* + * If the user set a custom pid_max limit we could be + * in the millions. + * Skip the test in this case. + */ + if (recycled_pid > PIDFD_MAX_DEFAULT) + _exit(PIDFD_SKIP); + } + + /* failed to recycle pid */ + _exit(PIDFD_SKIP); + } + + ret = wait_for_pid(pid1); + switch (ret) { + case PIDFD_FAIL: + ksft_exit_fail_msg( + "%s test: Managed to signal recycled pid %d\n", + test_name, PID_RECYCLE); + case PIDFD_PASS: + ksft_exit_fail_msg("%s test: Failed to recycle pid %d\n", + test_name, PID_RECYCLE); + case PIDFD_SKIP: + ksft_print_msg("%s test: Skipping test\n", test_name); + ret = 0; + break; + case PIDFD_XFAIL: + ksft_test_result_pass( + "%s test: Failed to signal recycled pid as expected\n", + test_name); + ret = 0; + break; + default /* PIDFD_ERROR */: + ksft_exit_fail_msg("%s test: Error while running tests\n", + test_name); + } + + return ret; +} + +static int test_pidfd_send_signal_syscall_support(void) +{ + int pidfd, ret; + const char *test_name = "pidfd_send_signal check for support"; + + pidfd = open("/proc/self", O_DIRECTORY | O_CLOEXEC); + if (pidfd < 0) + ksft_exit_fail_msg( + "%s test: Failed to open process file descriptor\n", + test_name); + + ret = sys_pidfd_send_signal(pidfd, 0, NULL, 0); + if (ret < 0) { + /* + * pidfd_send_signal() will currently return ENOSYS when + * CONFIG_PROC_FS is not set. + */ + if (errno == ENOSYS) + ksft_exit_skip( + "%s test: pidfd_send_signal() syscall not supported (Ensure that CONFIG_PROC_FS=y is set)\n", + test_name); + + ksft_exit_fail_msg("%s test: Failed to send signal\n", + test_name); + } + + close(pidfd); + ksft_test_result_pass( + "%s test: pidfd_send_signal() syscall is supported. Tests can be executed\n", + test_name); + return 0; +} + +int main(int argc, char **argv) +{ + ksft_print_header(); + + test_pidfd_send_signal_syscall_support(); + test_pidfd_send_signal_simple_success(); + test_pidfd_send_signal_exited_fail(); + test_pidfd_send_signal_recycled_pid_fail(); + + return ksft_exit_pass(); +} diff --git a/tools/testing/selftests/proc/proc-pid-vm.c b/tools/testing/selftests/proc/proc-pid-vm.c index bbe8150d18aaa..7202bbac976ea 100644 --- a/tools/testing/selftests/proc/proc-pid-vm.c +++ b/tools/testing/selftests/proc/proc-pid-vm.c @@ -29,6 +29,7 @@ #include <errno.h> #include <sched.h> #include <signal.h> +#include <stdbool.h> #include <stdint.h> #include <stdio.h> #include <string.h> @@ -36,11 +37,14 @@ #include <sys/mount.h> #include <sys/types.h> #include <sys/stat.h> +#include <sys/wait.h> #include <fcntl.h> #include <unistd.h> #include <sys/syscall.h> #include <sys/uio.h> #include <linux/kdev_t.h> +#include <sys/time.h> +#include <sys/resource.h> static inline long sys_execveat(int dirfd, const char *pathname, char **argv, char **envp, int flags) { @@ -205,12 +209,44 @@ static int make_exe(const uint8_t *payload, size_t len) } #endif +static bool g_vsyscall = false; + +static const char str_vsyscall[] = +"ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]\n"; + #ifdef __x86_64__ +/* + * vsyscall page can't be unmapped, probe it with memory load. + */ +static void vsyscall(void) +{ + pid_t pid; + int wstatus; + + pid = fork(); + if (pid < 0) { + fprintf(stderr, "fork, errno %d\n", errno); + exit(1); + } + if (pid == 0) { + struct rlimit rlim = {0, 0}; + (void)setrlimit(RLIMIT_CORE, &rlim); + *(volatile int *)0xffffffffff600000UL; + exit(0); + } + wait(&wstatus); + if (WIFEXITED(wstatus)) { + g_vsyscall = true; + } +} + int main(void) { int pipefd[2]; int exec_fd; + vsyscall(); + atexit(ate); make_private_tmp(); @@ -261,9 +297,9 @@ int main(void) snprintf(buf0 + MAPS_OFFSET, sizeof(buf0) - MAPS_OFFSET, "/tmp/#%llu (deleted)\n", (unsigned long long)st.st_ino); - /* Test /proc/$PID/maps */ { + const size_t len = strlen(buf0) + (g_vsyscall ? strlen(str_vsyscall) : 0); char buf[256]; ssize_t rv; int fd; @@ -274,13 +310,16 @@ int main(void) return 1; } rv = read(fd, buf, sizeof(buf)); - assert(rv == strlen(buf0)); + assert(rv == len); assert(memcmp(buf, buf0, strlen(buf0)) == 0); + if (g_vsyscall) { + assert(memcmp(buf + strlen(buf0), str_vsyscall, strlen(str_vsyscall)) == 0); + } } /* Test /proc/$PID/smaps */ { - char buf[1024]; + char buf[4096]; ssize_t rv; int fd; @@ -319,6 +358,10 @@ int main(void) for (i = 0; i < sizeof(S)/sizeof(S[0]); i++) { assert(memmem(buf, rv, S[i], strlen(S[i]))); } + + if (g_vsyscall) { + assert(memmem(buf, rv, str_vsyscall, strlen(str_vsyscall))); + } } /* Test /proc/$PID/smaps_rollup */ diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c index b07ac4614e1c6..3417f2dbc3667 100644 --- a/virt/kvm/arm/arch_timer.c +++ b/virt/kvm/arm/arch_timer.c @@ -25,6 +25,7 @@ #include <clocksource/arm_arch_timer.h> #include <asm/arch_timer.h> +#include <asm/kvm_emulate.h> #include <asm/kvm_hyp.h> #include <kvm/arm_vgic.h> @@ -34,7 +35,9 @@ static struct timecounter *timecounter; static unsigned int host_vtimer_irq; +static unsigned int host_ptimer_irq; static u32 host_vtimer_irq_flags; +static u32 host_ptimer_irq_flags; static DEFINE_STATIC_KEY_FALSE(has_gic_active_state); @@ -52,12 +55,34 @@ static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx); static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, struct arch_timer_context *timer_ctx); static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx); +static void kvm_arm_timer_write(struct kvm_vcpu *vcpu, + struct arch_timer_context *timer, + enum kvm_arch_timer_regs treg, + u64 val); +static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu, + struct arch_timer_context *timer, + enum kvm_arch_timer_regs treg); u64 kvm_phys_timer_read(void) { return timecounter->cc->read(timecounter->cc); } +static void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map) +{ + if (has_vhe()) { + map->direct_vtimer = vcpu_vtimer(vcpu); + map->direct_ptimer = vcpu_ptimer(vcpu); + map->emul_ptimer = NULL; + } else { + map->direct_vtimer = vcpu_vtimer(vcpu); + map->direct_ptimer = NULL; + map->emul_ptimer = vcpu_ptimer(vcpu); + } + + trace_kvm_get_timer_map(vcpu->vcpu_id, map); +} + static inline bool userspace_irqchip(struct kvm *kvm) { return static_branch_unlikely(&userspace_irqchip_in_use) && @@ -78,20 +103,27 @@ static void soft_timer_cancel(struct hrtimer *hrt) static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id) { struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id; - struct arch_timer_context *vtimer; + struct arch_timer_context *ctx; + struct timer_map map; /* * We may see a timer interrupt after vcpu_put() has been called which * sets the CPU's vcpu pointer to NULL, because even though the timer - * has been disabled in vtimer_save_state(), the hardware interrupt + * has been disabled in timer_save_state(), the hardware interrupt * signal may not have been retired from the interrupt controller yet. */ if (!vcpu) return IRQ_HANDLED; - vtimer = vcpu_vtimer(vcpu); - if (kvm_timer_should_fire(vtimer)) - kvm_timer_update_irq(vcpu, true, vtimer); + get_timer_map(vcpu, &map); + + if (irq == host_vtimer_irq) + ctx = map.direct_vtimer; + else + ctx = map.direct_ptimer; + + if (kvm_timer_should_fire(ctx)) + kvm_timer_update_irq(vcpu, true, ctx); if (userspace_irqchip(vcpu->kvm) && !static_branch_unlikely(&has_gic_active_state)) @@ -122,7 +154,9 @@ static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx) static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) { - return !(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_IT_MASK) && + WARN_ON(timer_ctx && timer_ctx->loaded); + return timer_ctx && + !(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_IT_MASK) && (timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_ENABLE); } @@ -132,21 +166,22 @@ static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) */ static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu) { - u64 min_virt = ULLONG_MAX, min_phys = ULLONG_MAX; - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); + u64 min_delta = ULLONG_MAX; + int i; - if (kvm_timer_irq_can_fire(vtimer)) - min_virt = kvm_timer_compute_delta(vtimer); + for (i = 0; i < NR_KVM_TIMERS; i++) { + struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i]; - if (kvm_timer_irq_can_fire(ptimer)) - min_phys = kvm_timer_compute_delta(ptimer); + WARN(ctx->loaded, "timer %d loaded\n", i); + if (kvm_timer_irq_can_fire(ctx)) + min_delta = min(min_delta, kvm_timer_compute_delta(ctx)); + } /* If none of timers can fire, then return 0 */ - if ((min_virt == ULLONG_MAX) && (min_phys == ULLONG_MAX)) + if (min_delta == ULLONG_MAX) return 0; - return min(min_virt, min_phys); + return min_delta; } static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt) @@ -173,41 +208,58 @@ static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt) return HRTIMER_NORESTART; } -static enum hrtimer_restart kvm_phys_timer_expire(struct hrtimer *hrt) +static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt) { - struct arch_timer_context *ptimer; - struct arch_timer_cpu *timer; + struct arch_timer_context *ctx; struct kvm_vcpu *vcpu; u64 ns; - timer = container_of(hrt, struct arch_timer_cpu, phys_timer); - vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu); - ptimer = vcpu_ptimer(vcpu); + ctx = container_of(hrt, struct arch_timer_context, hrtimer); + vcpu = ctx->vcpu; + + trace_kvm_timer_hrtimer_expire(ctx); /* * Check that the timer has really expired from the guest's * PoV (NTP on the host may have forced it to expire * early). If not ready, schedule for a later time. */ - ns = kvm_timer_compute_delta(ptimer); + ns = kvm_timer_compute_delta(ctx); if (unlikely(ns)) { hrtimer_forward_now(hrt, ns_to_ktime(ns)); return HRTIMER_RESTART; } - kvm_timer_update_irq(vcpu, true, ptimer); + kvm_timer_update_irq(vcpu, true, ctx); return HRTIMER_NORESTART; } static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx) { + enum kvm_arch_timers index; u64 cval, now; + if (!timer_ctx) + return false; + + index = arch_timer_ctx_index(timer_ctx); + if (timer_ctx->loaded) { - u32 cnt_ctl; + u32 cnt_ctl = 0; + + switch (index) { + case TIMER_VTIMER: + cnt_ctl = read_sysreg_el0(cntv_ctl); + break; + case TIMER_PTIMER: + cnt_ctl = read_sysreg_el0(cntp_ctl); + break; + case NR_KVM_TIMERS: + /* GCC is braindead */ + cnt_ctl = 0; + break; + } - /* Only the virtual timer can be loaded so far */ - cnt_ctl = read_sysreg_el0(cntv_ctl); return (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) && (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) && !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK); @@ -224,13 +276,13 @@ static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx) bool kvm_timer_is_pending(struct kvm_vcpu *vcpu) { - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); + struct timer_map map; - if (kvm_timer_should_fire(vtimer)) - return true; + get_timer_map(vcpu, &map); - return kvm_timer_should_fire(ptimer); + return kvm_timer_should_fire(map.direct_vtimer) || + kvm_timer_should_fire(map.direct_ptimer) || + kvm_timer_should_fire(map.emul_ptimer); } /* @@ -269,77 +321,70 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, } } -/* Schedule the background timer for the emulated timer. */ -static void phys_timer_emulate(struct kvm_vcpu *vcpu) +static void timer_emulate(struct arch_timer_context *ctx) { - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); + bool should_fire = kvm_timer_should_fire(ctx); + + trace_kvm_timer_emulate(ctx, should_fire); + + if (should_fire) { + kvm_timer_update_irq(ctx->vcpu, true, ctx); + return; + } /* * If the timer can fire now, we don't need to have a soft timer * scheduled for the future. If the timer cannot fire at all, * then we also don't need a soft timer. */ - if (kvm_timer_should_fire(ptimer) || !kvm_timer_irq_can_fire(ptimer)) { - soft_timer_cancel(&timer->phys_timer); + if (!kvm_timer_irq_can_fire(ctx)) { + soft_timer_cancel(&ctx->hrtimer); return; } - soft_timer_start(&timer->phys_timer, kvm_timer_compute_delta(ptimer)); + soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx)); } -/* - * Check if there was a change in the timer state, so that we should either - * raise or lower the line level to the GIC or schedule a background timer to - * emulate the physical timer. - */ -static void kvm_timer_update_state(struct kvm_vcpu *vcpu) +static void timer_save_state(struct arch_timer_context *ctx) { - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); - bool level; + struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu); + enum kvm_arch_timers index = arch_timer_ctx_index(ctx); + unsigned long flags; - if (unlikely(!timer->enabled)) + if (!timer->enabled) return; - /* - * The vtimer virtual interrupt is a 'mapped' interrupt, meaning part - * of its lifecycle is offloaded to the hardware, and we therefore may - * not have lowered the irq.level value before having to signal a new - * interrupt, but have to signal an interrupt every time the level is - * asserted. - */ - level = kvm_timer_should_fire(vtimer); - kvm_timer_update_irq(vcpu, level, vtimer); + local_irq_save(flags); - phys_timer_emulate(vcpu); + if (!ctx->loaded) + goto out; - if (kvm_timer_should_fire(ptimer) != ptimer->irq.level) - kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer); -} + switch (index) { + case TIMER_VTIMER: + ctx->cnt_ctl = read_sysreg_el0(cntv_ctl); + ctx->cnt_cval = read_sysreg_el0(cntv_cval); -static void vtimer_save_state(struct kvm_vcpu *vcpu) -{ - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - unsigned long flags; + /* Disable the timer */ + write_sysreg_el0(0, cntv_ctl); + isb(); - local_irq_save(flags); + break; + case TIMER_PTIMER: + ctx->cnt_ctl = read_sysreg_el0(cntp_ctl); + ctx->cnt_cval = read_sysreg_el0(cntp_cval); - if (!vtimer->loaded) - goto out; + /* Disable the timer */ + write_sysreg_el0(0, cntp_ctl); + isb(); - if (timer->enabled) { - vtimer->cnt_ctl = read_sysreg_el0(cntv_ctl); - vtimer->cnt_cval = read_sysreg_el0(cntv_cval); + break; + case NR_KVM_TIMERS: + BUG(); } - /* Disable the virtual timer */ - write_sysreg_el0(0, cntv_ctl); - isb(); + trace_kvm_timer_save_state(ctx); - vtimer->loaded = false; + ctx->loaded = false; out: local_irq_restore(flags); } @@ -349,67 +394,72 @@ out: * thread is removed from its waitqueue and made runnable when there's a timer * interrupt to handle. */ -void kvm_timer_schedule(struct kvm_vcpu *vcpu) +static void kvm_timer_blocking(struct kvm_vcpu *vcpu) { - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); - - vtimer_save_state(vcpu); + struct arch_timer_cpu *timer = vcpu_timer(vcpu); + struct timer_map map; - /* - * No need to schedule a background timer if any guest timer has - * already expired, because kvm_vcpu_block will return before putting - * the thread to sleep. - */ - if (kvm_timer_should_fire(vtimer) || kvm_timer_should_fire(ptimer)) - return; + get_timer_map(vcpu, &map); /* - * If both timers are not capable of raising interrupts (disabled or + * If no timers are capable of raising interrupts (disabled or * masked), then there's no more work for us to do. */ - if (!kvm_timer_irq_can_fire(vtimer) && !kvm_timer_irq_can_fire(ptimer)) + if (!kvm_timer_irq_can_fire(map.direct_vtimer) && + !kvm_timer_irq_can_fire(map.direct_ptimer) && + !kvm_timer_irq_can_fire(map.emul_ptimer)) return; /* - * The guest timers have not yet expired, schedule a background timer. + * At least one guest time will expire. Schedule a background timer. * Set the earliest expiration time among the guest timers. */ soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu)); } -static void vtimer_restore_state(struct kvm_vcpu *vcpu) +static void kvm_timer_unblocking(struct kvm_vcpu *vcpu) { - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); + struct arch_timer_cpu *timer = vcpu_timer(vcpu); + + soft_timer_cancel(&timer->bg_timer); +} + +static void timer_restore_state(struct arch_timer_context *ctx) +{ + struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu); + enum kvm_arch_timers index = arch_timer_ctx_index(ctx); unsigned long flags; + if (!timer->enabled) + return; + local_irq_save(flags); - if (vtimer->loaded) + if (ctx->loaded) goto out; - if (timer->enabled) { - write_sysreg_el0(vtimer->cnt_cval, cntv_cval); + switch (index) { + case TIMER_VTIMER: + write_sysreg_el0(ctx->cnt_cval, cntv_cval); isb(); - write_sysreg_el0(vtimer->cnt_ctl, cntv_ctl); + write_sysreg_el0(ctx->cnt_ctl, cntv_ctl); + break; + case TIMER_PTIMER: + write_sysreg_el0(ctx->cnt_cval, cntp_cval); + isb(); + write_sysreg_el0(ctx->cnt_ctl, cntp_ctl); + break; + case NR_KVM_TIMERS: + BUG(); } - vtimer->loaded = true; + trace_kvm_timer_restore_state(ctx); + + ctx->loaded = true; out: local_irq_restore(flags); } -void kvm_timer_unschedule(struct kvm_vcpu *vcpu) -{ - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; - - vtimer_restore_state(vcpu); - - soft_timer_cancel(&timer->bg_timer); -} - static void set_cntvoff(u64 cntvoff) { u32 low = lower_32_bits(cntvoff); @@ -425,23 +475,32 @@ static void set_cntvoff(u64 cntvoff) kvm_call_hyp(__kvm_timer_set_cntvoff, low, high); } -static inline void set_vtimer_irq_phys_active(struct kvm_vcpu *vcpu, bool active) +static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, bool active) { int r; - r = irq_set_irqchip_state(host_vtimer_irq, IRQCHIP_STATE_ACTIVE, active); + r = irq_set_irqchip_state(ctx->host_timer_irq, IRQCHIP_STATE_ACTIVE, active); WARN_ON(r); } -static void kvm_timer_vcpu_load_gic(struct kvm_vcpu *vcpu) +static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx) { - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - bool phys_active; + struct kvm_vcpu *vcpu = ctx->vcpu; + bool phys_active = false; + + /* + * Update the timer output so that it is likely to match the + * state we're about to restore. If the timer expires between + * this point and the register restoration, we'll take the + * interrupt anyway. + */ + kvm_timer_update_irq(ctx->vcpu, kvm_timer_should_fire(ctx), ctx); if (irqchip_in_kernel(vcpu->kvm)) - phys_active = kvm_vgic_map_is_active(vcpu, vtimer->irq.irq); - else - phys_active = vtimer->irq.level; - set_vtimer_irq_phys_active(vcpu, phys_active); + phys_active = kvm_vgic_map_is_active(vcpu, ctx->irq.irq); + + phys_active |= ctx->irq.level; + + set_timer_irq_phys_active(ctx, phys_active); } static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu) @@ -466,28 +525,32 @@ static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu) void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu) { - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); + struct arch_timer_cpu *timer = vcpu_timer(vcpu); + struct timer_map map; if (unlikely(!timer->enabled)) return; - if (static_branch_likely(&has_gic_active_state)) - kvm_timer_vcpu_load_gic(vcpu); - else + get_timer_map(vcpu, &map); + + if (static_branch_likely(&has_gic_active_state)) { + kvm_timer_vcpu_load_gic(map.direct_vtimer); + if (map.direct_ptimer) + kvm_timer_vcpu_load_gic(map.direct_ptimer); + } else { kvm_timer_vcpu_load_nogic(vcpu); + } - set_cntvoff(vtimer->cntvoff); + set_cntvoff(map.direct_vtimer->cntvoff); - vtimer_restore_state(vcpu); + kvm_timer_unblocking(vcpu); - /* Set the background timer for the physical timer emulation. */ - phys_timer_emulate(vcpu); + timer_restore_state(map.direct_vtimer); + if (map.direct_ptimer) + timer_restore_state(map.direct_ptimer); - /* If the timer fired while we weren't running, inject it now */ - if (kvm_timer_should_fire(ptimer) != ptimer->irq.level) - kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer); + if (map.emul_ptimer) + timer_emulate(map.emul_ptimer); } bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu) @@ -509,15 +572,20 @@ bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu) void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) { - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; + struct arch_timer_cpu *timer = vcpu_timer(vcpu); + struct timer_map map; if (unlikely(!timer->enabled)) return; - vtimer_save_state(vcpu); + get_timer_map(vcpu, &map); + + timer_save_state(map.direct_vtimer); + if (map.direct_ptimer) + timer_save_state(map.direct_ptimer); /* - * Cancel the physical timer emulation, because the only case where we + * Cancel soft timer emulation, because the only case where we * need it after a vcpu_put is in the context of a sleeping VCPU, and * in that case we already factor in the deadline for the physical * timer when scheduling the bg_timer. @@ -525,7 +593,11 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) * In any case, we re-schedule the hrtimer for the physical timer when * coming back to the VCPU thread in kvm_timer_vcpu_load(). */ - soft_timer_cancel(&timer->phys_timer); + if (map.emul_ptimer) + soft_timer_cancel(&map.emul_ptimer->hrtimer); + + if (swait_active(kvm_arch_vcpu_wq(vcpu))) + kvm_timer_blocking(vcpu); /* * The kernel may decide to run userspace after calling vcpu_put, so @@ -534,8 +606,7 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) * counter of non-VHE case. For VHE, the virtual counter uses a fixed * virtual offset of zero, so no need to zero CNTVOFF_EL2 register. */ - if (!has_vhe()) - set_cntvoff(0); + set_cntvoff(0); } /* @@ -550,7 +621,7 @@ static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu) if (!kvm_timer_should_fire(vtimer)) { kvm_timer_update_irq(vcpu, false, vtimer); if (static_branch_likely(&has_gic_active_state)) - set_vtimer_irq_phys_active(vcpu, false); + set_timer_irq_phys_active(vtimer, false); else enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); } @@ -558,7 +629,7 @@ static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu) void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu) { - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; + struct arch_timer_cpu *timer = vcpu_timer(vcpu); if (unlikely(!timer->enabled)) return; @@ -569,9 +640,10 @@ void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu) int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu) { - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); + struct arch_timer_cpu *timer = vcpu_timer(vcpu); + struct timer_map map; + + get_timer_map(vcpu, &map); /* * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8 @@ -579,12 +651,22 @@ int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu) * resets the timer to be disabled and unmasked and is compliant with * the ARMv7 architecture. */ - vtimer->cnt_ctl = 0; - ptimer->cnt_ctl = 0; - kvm_timer_update_state(vcpu); + vcpu_vtimer(vcpu)->cnt_ctl = 0; + vcpu_ptimer(vcpu)->cnt_ctl = 0; - if (timer->enabled && irqchip_in_kernel(vcpu->kvm)) - kvm_vgic_reset_mapped_irq(vcpu, vtimer->irq.irq); + if (timer->enabled) { + kvm_timer_update_irq(vcpu, false, vcpu_vtimer(vcpu)); + kvm_timer_update_irq(vcpu, false, vcpu_ptimer(vcpu)); + + if (irqchip_in_kernel(vcpu->kvm)) { + kvm_vgic_reset_mapped_irq(vcpu, map.direct_vtimer->irq.irq); + if (map.direct_ptimer) + kvm_vgic_reset_mapped_irq(vcpu, map.direct_ptimer->irq.irq); + } + } + + if (map.emul_ptimer) + soft_timer_cancel(&map.emul_ptimer->hrtimer); return 0; } @@ -610,56 +692,76 @@ static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff) void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu) { - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; + struct arch_timer_cpu *timer = vcpu_timer(vcpu); struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); /* Synchronize cntvoff across all vtimers of a VM. */ update_vtimer_cntvoff(vcpu, kvm_phys_timer_read()); - vcpu_ptimer(vcpu)->cntvoff = 0; + ptimer->cntvoff = 0; hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); timer->bg_timer.function = kvm_bg_timer_expire; - hrtimer_init(&timer->phys_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); - timer->phys_timer.function = kvm_phys_timer_expire; + hrtimer_init(&vtimer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + hrtimer_init(&ptimer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + vtimer->hrtimer.function = kvm_hrtimer_expire; + ptimer->hrtimer.function = kvm_hrtimer_expire; vtimer->irq.irq = default_vtimer_irq.irq; ptimer->irq.irq = default_ptimer_irq.irq; + + vtimer->host_timer_irq = host_vtimer_irq; + ptimer->host_timer_irq = host_ptimer_irq; + + vtimer->host_timer_irq_flags = host_vtimer_irq_flags; + ptimer->host_timer_irq_flags = host_ptimer_irq_flags; + + vtimer->vcpu = vcpu; + ptimer->vcpu = vcpu; } static void kvm_timer_init_interrupt(void *info) { enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); + enable_percpu_irq(host_ptimer_irq, host_ptimer_irq_flags); } int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value) { - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); + struct arch_timer_context *timer; + bool level; switch (regid) { case KVM_REG_ARM_TIMER_CTL: - vtimer->cnt_ctl = value & ~ARCH_TIMER_CTRL_IT_STAT; + timer = vcpu_vtimer(vcpu); + kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value); break; case KVM_REG_ARM_TIMER_CNT: + timer = vcpu_vtimer(vcpu); update_vtimer_cntvoff(vcpu, kvm_phys_timer_read() - value); break; case KVM_REG_ARM_TIMER_CVAL: - vtimer->cnt_cval = value; + timer = vcpu_vtimer(vcpu); + kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value); break; case KVM_REG_ARM_PTIMER_CTL: - ptimer->cnt_ctl = value & ~ARCH_TIMER_CTRL_IT_STAT; + timer = vcpu_ptimer(vcpu); + kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value); break; case KVM_REG_ARM_PTIMER_CVAL: - ptimer->cnt_cval = value; + timer = vcpu_ptimer(vcpu); + kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value); break; default: return -1; } - kvm_timer_update_state(vcpu); + level = kvm_timer_should_fire(timer); + kvm_timer_update_irq(vcpu, level, timer); + timer_emulate(timer); + return 0; } @@ -679,26 +781,113 @@ static u64 read_timer_ctl(struct arch_timer_context *timer) u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid) { - struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); - switch (regid) { case KVM_REG_ARM_TIMER_CTL: - return read_timer_ctl(vtimer); + return kvm_arm_timer_read(vcpu, + vcpu_vtimer(vcpu), TIMER_REG_CTL); case KVM_REG_ARM_TIMER_CNT: - return kvm_phys_timer_read() - vtimer->cntvoff; + return kvm_arm_timer_read(vcpu, + vcpu_vtimer(vcpu), TIMER_REG_CNT); case KVM_REG_ARM_TIMER_CVAL: - return vtimer->cnt_cval; + return kvm_arm_timer_read(vcpu, + vcpu_vtimer(vcpu), TIMER_REG_CVAL); case KVM_REG_ARM_PTIMER_CTL: - return read_timer_ctl(ptimer); - case KVM_REG_ARM_PTIMER_CVAL: - return ptimer->cnt_cval; + return kvm_arm_timer_read(vcpu, + vcpu_ptimer(vcpu), TIMER_REG_CTL); case KVM_REG_ARM_PTIMER_CNT: - return kvm_phys_timer_read(); + return kvm_arm_timer_read(vcpu, + vcpu_vtimer(vcpu), TIMER_REG_CNT); + case KVM_REG_ARM_PTIMER_CVAL: + return kvm_arm_timer_read(vcpu, + vcpu_ptimer(vcpu), TIMER_REG_CVAL); } return (u64)-1; } +static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu, + struct arch_timer_context *timer, + enum kvm_arch_timer_regs treg) +{ + u64 val; + + switch (treg) { + case TIMER_REG_TVAL: + val = kvm_phys_timer_read() - timer->cntvoff - timer->cnt_cval; + break; + + case TIMER_REG_CTL: + val = read_timer_ctl(timer); + break; + + case TIMER_REG_CVAL: + val = timer->cnt_cval; + break; + + case TIMER_REG_CNT: + val = kvm_phys_timer_read() - timer->cntvoff; + break; + + default: + BUG(); + } + + return val; +} + +u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu, + enum kvm_arch_timers tmr, + enum kvm_arch_timer_regs treg) +{ + u64 val; + + preempt_disable(); + kvm_timer_vcpu_put(vcpu); + + val = kvm_arm_timer_read(vcpu, vcpu_get_timer(vcpu, tmr), treg); + + kvm_timer_vcpu_load(vcpu); + preempt_enable(); + + return val; +} + +static void kvm_arm_timer_write(struct kvm_vcpu *vcpu, + struct arch_timer_context *timer, + enum kvm_arch_timer_regs treg, + u64 val) +{ + switch (treg) { + case TIMER_REG_TVAL: + timer->cnt_cval = val - kvm_phys_timer_read() - timer->cntvoff; + break; + + case TIMER_REG_CTL: + timer->cnt_ctl = val & ~ARCH_TIMER_CTRL_IT_STAT; + break; + + case TIMER_REG_CVAL: + timer->cnt_cval = val; + break; + + default: + BUG(); + } +} + +void kvm_arm_timer_write_sysreg(struct kvm_vcpu *vcpu, + enum kvm_arch_timers tmr, + enum kvm_arch_timer_regs treg, + u64 val) +{ + preempt_disable(); + kvm_timer_vcpu_put(vcpu); + + kvm_arm_timer_write(vcpu, vcpu_get_timer(vcpu, tmr), treg, val); + + kvm_timer_vcpu_load(vcpu); + preempt_enable(); +} + static int kvm_timer_starting_cpu(unsigned int cpu) { kvm_timer_init_interrupt(NULL); @@ -724,6 +913,8 @@ int kvm_timer_hyp_init(bool has_gic) return -ENODEV; } + /* First, do the virtual EL1 timer irq */ + if (info->virtual_irq <= 0) { kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n", info->virtual_irq); @@ -734,15 +925,15 @@ int kvm_timer_hyp_init(bool has_gic) host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq); if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH && host_vtimer_irq_flags != IRQF_TRIGGER_LOW) { - kvm_err("Invalid trigger for IRQ%d, assuming level low\n", + kvm_err("Invalid trigger for vtimer IRQ%d, assuming level low\n", host_vtimer_irq); host_vtimer_irq_flags = IRQF_TRIGGER_LOW; } err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler, - "kvm guest timer", kvm_get_running_vcpus()); + "kvm guest vtimer", kvm_get_running_vcpus()); if (err) { - kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n", + kvm_err("kvm_arch_timer: can't request vtimer interrupt %d (%d)\n", host_vtimer_irq, err); return err; } @@ -760,6 +951,43 @@ int kvm_timer_hyp_init(bool has_gic) kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq); + /* Now let's do the physical EL1 timer irq */ + + if (info->physical_irq > 0) { + host_ptimer_irq = info->physical_irq; + host_ptimer_irq_flags = irq_get_trigger_type(host_ptimer_irq); + if (host_ptimer_irq_flags != IRQF_TRIGGER_HIGH && + host_ptimer_irq_flags != IRQF_TRIGGER_LOW) { + kvm_err("Invalid trigger for ptimer IRQ%d, assuming level low\n", + host_ptimer_irq); + host_ptimer_irq_flags = IRQF_TRIGGER_LOW; + } + + err = request_percpu_irq(host_ptimer_irq, kvm_arch_timer_handler, + "kvm guest ptimer", kvm_get_running_vcpus()); + if (err) { + kvm_err("kvm_arch_timer: can't request ptimer interrupt %d (%d)\n", + host_ptimer_irq, err); + return err; + } + + if (has_gic) { + err = irq_set_vcpu_affinity(host_ptimer_irq, + kvm_get_running_vcpus()); + if (err) { + kvm_err("kvm_arch_timer: error setting vcpu affinity\n"); + goto out_free_irq; + } + } + + kvm_debug("physical timer IRQ%d\n", host_ptimer_irq); + } else if (has_vhe()) { + kvm_err("kvm_arch_timer: invalid physical timer IRQ: %d\n", + info->physical_irq); + err = -ENODEV; + goto out_free_irq; + } + cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING, "kvm/arm/timer:starting", kvm_timer_starting_cpu, kvm_timer_dying_cpu); @@ -771,7 +999,7 @@ out_free_irq: void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu) { - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; + struct arch_timer_cpu *timer = vcpu_timer(vcpu); soft_timer_cancel(&timer->bg_timer); } @@ -807,16 +1035,18 @@ bool kvm_arch_timer_get_input_level(int vintid) if (vintid == vcpu_vtimer(vcpu)->irq.irq) timer = vcpu_vtimer(vcpu); + else if (vintid == vcpu_ptimer(vcpu)->irq.irq) + timer = vcpu_ptimer(vcpu); else - BUG(); /* We only map the vtimer so far */ + BUG(); return kvm_timer_should_fire(timer); } int kvm_timer_enable(struct kvm_vcpu *vcpu) { - struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; - struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); + struct arch_timer_cpu *timer = vcpu_timer(vcpu); + struct timer_map map; int ret; if (timer->enabled) @@ -834,19 +1064,33 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu) return -EINVAL; } - ret = kvm_vgic_map_phys_irq(vcpu, host_vtimer_irq, vtimer->irq.irq, + get_timer_map(vcpu, &map); + + ret = kvm_vgic_map_phys_irq(vcpu, + map.direct_vtimer->host_timer_irq, + map.direct_vtimer->irq.irq, kvm_arch_timer_get_input_level); if (ret) return ret; + if (map.direct_ptimer) { + ret = kvm_vgic_map_phys_irq(vcpu, + map.direct_ptimer->host_timer_irq, + map.direct_ptimer->irq.irq, + kvm_arch_timer_get_input_level); + } + + if (ret) + return ret; + no_vgic: timer->enabled = 1; return 0; } /* - * On VHE system, we only need to configure trap on physical timer and counter - * accesses in EL0 and EL1 once, not for every world switch. + * On VHE system, we only need to configure the EL2 timer trap register once, + * not for every world switch. * The host kernel runs at EL2 with HCR_EL2.TGE == 1, * and this makes those bits have no effect for the host kernel execution. */ @@ -857,11 +1101,11 @@ void kvm_timer_init_vhe(void) u64 val; /* - * Disallow physical timer access for the guest. - * Physical counter access is allowed. + * VHE systems allow the guest direct access to the EL1 physical + * timer/counter. */ val = read_sysreg(cnthctl_el2); - val &= ~(CNTHCTL_EL1PCEN << cnthctl_shift); + val |= (CNTHCTL_EL1PCEN << cnthctl_shift); val |= (CNTHCTL_EL1PCTEN << cnthctl_shift); write_sysreg(val, cnthctl_el2); } diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c index 9c486fad3f9f8..99c37384ba7bd 100644 --- a/virt/kvm/arm/arm.c +++ b/virt/kvm/arm/arm.c @@ -65,7 +65,6 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu); /* The VMID used in the VTTBR */ static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1); static u32 kvm_next_vmid; -static unsigned int kvm_vmid_bits __read_mostly; static DEFINE_SPINLOCK(kvm_vmid_lock); static bool vgic_present; @@ -142,7 +141,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm_vgic_early_init(kvm); /* Mark the initial VMID generation invalid */ - kvm->arch.vmid_gen = 0; + kvm->arch.vmid.vmid_gen = 0; /* The maximum number of VCPUs is limited by the host's GIC model */ kvm->arch.max_vcpus = vgic_present ? @@ -336,13 +335,11 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) { - kvm_timer_schedule(vcpu); kvm_vgic_v4_enable_doorbell(vcpu); } void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) { - kvm_timer_unschedule(vcpu); kvm_vgic_v4_disable_doorbell(vcpu); } @@ -472,37 +469,31 @@ void force_vm_exit(const cpumask_t *mask) /** * need_new_vmid_gen - check that the VMID is still valid - * @kvm: The VM's VMID to check + * @vmid: The VMID to check * * return true if there is a new generation of VMIDs being used * - * The hardware supports only 256 values with the value zero reserved for the - * host, so we check if an assigned value belongs to a previous generation, - * which which requires us to assign a new value. If we're the first to use a - * VMID for the new generation, we must flush necessary caches and TLBs on all - * CPUs. + * The hardware supports a limited set of values with the value zero reserved + * for the host, so we check if an assigned value belongs to a previous + * generation, which which requires us to assign a new value. If we're the + * first to use a VMID for the new generation, we must flush necessary caches + * and TLBs on all CPUs. */ -static bool need_new_vmid_gen(struct kvm *kvm) +static bool need_new_vmid_gen(struct kvm_vmid *vmid) { u64 current_vmid_gen = atomic64_read(&kvm_vmid_gen); smp_rmb(); /* Orders read of kvm_vmid_gen and kvm->arch.vmid */ - return unlikely(READ_ONCE(kvm->arch.vmid_gen) != current_vmid_gen); + return unlikely(READ_ONCE(vmid->vmid_gen) != current_vmid_gen); } /** - * update_vttbr - Update the VTTBR with a valid VMID before the guest runs - * @kvm The guest that we are about to run - * - * Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the - * VM has a valid VMID, otherwise assigns a new one and flushes corresponding - * caches and TLBs. + * update_vmid - Update the vmid with a valid VMID for the current generation + * @kvm: The guest that struct vmid belongs to + * @vmid: The stage-2 VMID information struct */ -static void update_vttbr(struct kvm *kvm) +static void update_vmid(struct kvm_vmid *vmid) { - phys_addr_t pgd_phys; - u64 vmid, cnp = kvm_cpu_has_cnp() ? VTTBR_CNP_BIT : 0; - - if (!need_new_vmid_gen(kvm)) + if (!need_new_vmid_gen(vmid)) return; spin_lock(&kvm_vmid_lock); @@ -512,7 +503,7 @@ static void update_vttbr(struct kvm *kvm) * already allocated a valid vmid for this vm, then this vcpu should * use the same vmid. */ - if (!need_new_vmid_gen(kvm)) { + if (!need_new_vmid_gen(vmid)) { spin_unlock(&kvm_vmid_lock); return; } @@ -536,18 +527,12 @@ static void update_vttbr(struct kvm *kvm) kvm_call_hyp(__kvm_flush_vm_context); } - kvm->arch.vmid = kvm_next_vmid; + vmid->vmid = kvm_next_vmid; kvm_next_vmid++; - kvm_next_vmid &= (1 << kvm_vmid_bits) - 1; - - /* update vttbr to be used with the new vmid */ - pgd_phys = virt_to_phys(kvm->arch.pgd); - BUG_ON(pgd_phys & ~kvm_vttbr_baddr_mask(kvm)); - vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK(kvm_vmid_bits); - kvm->arch.vttbr = kvm_phys_to_vttbr(pgd_phys) | vmid | cnp; + kvm_next_vmid &= (1 << kvm_get_vmid_bits()) - 1; smp_wmb(); - WRITE_ONCE(kvm->arch.vmid_gen, atomic64_read(&kvm_vmid_gen)); + WRITE_ONCE(vmid->vmid_gen, atomic64_read(&kvm_vmid_gen)); spin_unlock(&kvm_vmid_lock); } @@ -700,7 +685,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) */ cond_resched(); - update_vttbr(vcpu->kvm); + update_vmid(&vcpu->kvm->arch.vmid); check_vcpu_requests(vcpu); @@ -749,7 +734,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) */ smp_store_mb(vcpu->mode, IN_GUEST_MODE); - if (ret <= 0 || need_new_vmid_gen(vcpu->kvm) || + if (ret <= 0 || need_new_vmid_gen(&vcpu->kvm->arch.vmid) || kvm_request_pending(vcpu)) { vcpu->mode = OUTSIDE_GUEST_MODE; isb(); /* Ensure work in x_flush_hwstate is committed */ @@ -775,7 +760,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) ret = kvm_vcpu_run_vhe(vcpu); kvm_arm_vhe_guest_exit(); } else { - ret = kvm_call_hyp(__kvm_vcpu_run_nvhe, vcpu); + ret = kvm_call_hyp_ret(__kvm_vcpu_run_nvhe, vcpu); } vcpu->mode = OUTSIDE_GUEST_MODE; @@ -1427,10 +1412,6 @@ static inline void hyp_cpu_pm_exit(void) static int init_common_resources(void) { - /* set size of VMID supported by CPU */ - kvm_vmid_bits = kvm_get_vmid_bits(); - kvm_info("%d-bit VMID\n", kvm_vmid_bits); - kvm_set_ipa_limit(); return 0; @@ -1571,6 +1552,7 @@ static int init_hyp_mode(void) kvm_cpu_context_t *cpu_ctxt; cpu_ctxt = per_cpu_ptr(&kvm_host_cpu_state, cpu); + kvm_init_host_cpu_context(cpu_ctxt, cpu); err = create_hyp_mappings(cpu_ctxt, cpu_ctxt + 1, PAGE_HYP); if (err) { @@ -1581,7 +1563,7 @@ static int init_hyp_mode(void) err = hyp_map_aux_data(); if (err) - kvm_err("Cannot map host auxilary data: %d\n", err); + kvm_err("Cannot map host auxiliary data: %d\n", err); return 0; diff --git a/virt/kvm/arm/hyp/vgic-v3-sr.c b/virt/kvm/arm/hyp/vgic-v3-sr.c index 9652c453480f5..264d92da32403 100644 --- a/virt/kvm/arm/hyp/vgic-v3-sr.c +++ b/virt/kvm/arm/hyp/vgic-v3-sr.c @@ -226,7 +226,7 @@ void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu) int i; u32 elrsr; - elrsr = read_gicreg(ICH_ELSR_EL2); + elrsr = read_gicreg(ICH_ELRSR_EL2); write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EN, ICH_HCR_EL2); diff --git a/virt/kvm/arm/mmu.c b/virt/kvm/arm/mmu.c index e9d28a7ca6734..ffd7acdceac73 100644 --- a/virt/kvm/arm/mmu.c +++ b/virt/kvm/arm/mmu.c @@ -908,6 +908,7 @@ int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size, */ int kvm_alloc_stage2_pgd(struct kvm *kvm) { + phys_addr_t pgd_phys; pgd_t *pgd; if (kvm->arch.pgd != NULL) { @@ -920,7 +921,12 @@ int kvm_alloc_stage2_pgd(struct kvm *kvm) if (!pgd) return -ENOMEM; + pgd_phys = virt_to_phys(pgd); + if (WARN_ON(pgd_phys & ~kvm_vttbr_baddr_mask(kvm))) + return -EINVAL; + kvm->arch.pgd = pgd; + kvm->arch.pgd_phys = pgd_phys; return 0; } @@ -1008,6 +1014,7 @@ void kvm_free_stage2_pgd(struct kvm *kvm) unmap_stage2_range(kvm, 0, kvm_phys_size(kvm)); pgd = READ_ONCE(kvm->arch.pgd); kvm->arch.pgd = NULL; + kvm->arch.pgd_phys = 0; } spin_unlock(&kvm->mmu_lock); @@ -1396,14 +1403,6 @@ static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, phys_addr_t *ipap) return false; } -static bool kvm_is_write_fault(struct kvm_vcpu *vcpu) -{ - if (kvm_vcpu_trap_is_iabt(vcpu)) - return false; - - return kvm_vcpu_dabt_iswrite(vcpu); -} - /** * stage2_wp_ptes - write protect PMD range * @pmd: pointer to pmd entry @@ -1598,14 +1597,13 @@ static void kvm_send_hwpoison_signal(unsigned long address, static bool fault_supports_stage2_pmd_mappings(struct kvm_memory_slot *memslot, unsigned long hva) { - gpa_t gpa_start, gpa_end; + gpa_t gpa_start; hva_t uaddr_start, uaddr_end; size_t size; size = memslot->npages * PAGE_SIZE; gpa_start = memslot->base_gfn << PAGE_SHIFT; - gpa_end = gpa_start + size; uaddr_start = memslot->userspace_addr; uaddr_end = uaddr_start + size; @@ -2353,7 +2351,7 @@ int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, return 0; } -void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots) +void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) { } diff --git a/virt/kvm/arm/trace.h b/virt/kvm/arm/trace.h index 3828beab93f26..204d210d01c29 100644 --- a/virt/kvm/arm/trace.h +++ b/virt/kvm/arm/trace.h @@ -2,6 +2,7 @@ #if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_KVM_H +#include <kvm/arm_arch_timer.h> #include <linux/tracepoint.h> #undef TRACE_SYSTEM @@ -262,10 +263,114 @@ TRACE_EVENT(kvm_timer_update_irq, __entry->vcpu_id, __entry->irq, __entry->level) ); +TRACE_EVENT(kvm_get_timer_map, + TP_PROTO(unsigned long vcpu_id, struct timer_map *map), + TP_ARGS(vcpu_id, map), + + TP_STRUCT__entry( + __field( unsigned long, vcpu_id ) + __field( int, direct_vtimer ) + __field( int, direct_ptimer ) + __field( int, emul_ptimer ) + ), + + TP_fast_assign( + __entry->vcpu_id = vcpu_id; + __entry->direct_vtimer = arch_timer_ctx_index(map->direct_vtimer); + __entry->direct_ptimer = + (map->direct_ptimer) ? arch_timer_ctx_index(map->direct_ptimer) : -1; + __entry->emul_ptimer = + (map->emul_ptimer) ? arch_timer_ctx_index(map->emul_ptimer) : -1; + ), + + TP_printk("VCPU: %ld, dv: %d, dp: %d, ep: %d", + __entry->vcpu_id, + __entry->direct_vtimer, + __entry->direct_ptimer, + __entry->emul_ptimer) +); + +TRACE_EVENT(kvm_timer_save_state, + TP_PROTO(struct arch_timer_context *ctx), + TP_ARGS(ctx), + + TP_STRUCT__entry( + __field( unsigned long, ctl ) + __field( unsigned long long, cval ) + __field( int, timer_idx ) + ), + + TP_fast_assign( + __entry->ctl = ctx->cnt_ctl; + __entry->cval = ctx->cnt_cval; + __entry->timer_idx = arch_timer_ctx_index(ctx); + ), + + TP_printk(" CTL: %#08lx CVAL: %#16llx arch_timer_ctx_index: %d", + __entry->ctl, + __entry->cval, + __entry->timer_idx) +); + +TRACE_EVENT(kvm_timer_restore_state, + TP_PROTO(struct arch_timer_context *ctx), + TP_ARGS(ctx), + + TP_STRUCT__entry( + __field( unsigned long, ctl ) + __field( unsigned long long, cval ) + __field( int, timer_idx ) + ), + + TP_fast_assign( + __entry->ctl = ctx->cnt_ctl; + __entry->cval = ctx->cnt_cval; + __entry->timer_idx = arch_timer_ctx_index(ctx); + ), + + TP_printk("CTL: %#08lx CVAL: %#16llx arch_timer_ctx_index: %d", + __entry->ctl, + __entry->cval, + __entry->timer_idx) +); + +TRACE_EVENT(kvm_timer_hrtimer_expire, + TP_PROTO(struct arch_timer_context *ctx), + TP_ARGS(ctx), + + TP_STRUCT__entry( + __field( int, timer_idx ) + ), + + TP_fast_assign( + __entry->timer_idx = arch_timer_ctx_index(ctx); + ), + + TP_printk("arch_timer_ctx_index: %d", __entry->timer_idx) +); + +TRACE_EVENT(kvm_timer_emulate, + TP_PROTO(struct arch_timer_context *ctx, bool should_fire), + TP_ARGS(ctx, should_fire), + + TP_STRUCT__entry( + __field( int, timer_idx ) + __field( bool, should_fire ) + ), + + TP_fast_assign( + __entry->timer_idx = arch_timer_ctx_index(ctx); + __entry->should_fire = should_fire; + ), + + TP_printk("arch_timer_ctx_index: %d (should_fire: %d)", + __entry->timer_idx, __entry->should_fire) +); + #endif /* _TRACE_KVM_H */ #undef TRACE_INCLUDE_PATH -#define TRACE_INCLUDE_PATH ../../../virt/kvm/arm +#define TRACE_INCLUDE_PATH ../../virt/kvm/arm #undef TRACE_INCLUDE_FILE #define TRACE_INCLUDE_FILE trace diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c index 4ee0aeb9a9058..408a78eb6a97b 100644 --- a/virt/kvm/arm/vgic/vgic-v3.c +++ b/virt/kvm/arm/vgic/vgic-v3.c @@ -589,7 +589,7 @@ early_param("kvm-arm.vgic_v4_enable", early_gicv4_enable); */ int vgic_v3_probe(const struct gic_kvm_info *info) { - u32 ich_vtr_el2 = kvm_call_hyp(__vgic_v3_get_ich_vtr_el2); + u32 ich_vtr_el2 = kvm_call_hyp_ret(__vgic_v3_get_ich_vtr_el2); int ret; /* @@ -679,7 +679,7 @@ void vgic_v3_put(struct kvm_vcpu *vcpu) struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; if (likely(cpu_if->vgic_sre)) - cpu_if->vgic_vmcr = kvm_call_hyp(__vgic_v3_read_vmcr); + cpu_if->vgic_vmcr = kvm_call_hyp_ret(__vgic_v3_read_vmcr); kvm_call_hyp(__vgic_v3_save_aprs, vcpu); diff --git a/virt/kvm/coalesced_mmio.c b/virt/kvm/coalesced_mmio.c index 6855cce3e5287..5294abb3f1788 100644 --- a/virt/kvm/coalesced_mmio.c +++ b/virt/kvm/coalesced_mmio.c @@ -144,7 +144,8 @@ int kvm_vm_ioctl_register_coalesced_mmio(struct kvm *kvm, if (zone->pio != 1 && zone->pio != 0) return -EINVAL; - dev = kzalloc(sizeof(struct kvm_coalesced_mmio_dev), GFP_KERNEL); + dev = kzalloc(sizeof(struct kvm_coalesced_mmio_dev), + GFP_KERNEL_ACCOUNT); if (!dev) return -ENOMEM; diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index b20b751286fc6..4325250afd728 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -297,7 +297,7 @@ kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args) if (!kvm_arch_intc_initialized(kvm)) return -EAGAIN; - irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL); + irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL_ACCOUNT); if (!irqfd) return -ENOMEM; @@ -345,7 +345,8 @@ kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args) } if (!irqfd->resampler) { - resampler = kzalloc(sizeof(*resampler), GFP_KERNEL); + resampler = kzalloc(sizeof(*resampler), + GFP_KERNEL_ACCOUNT); if (!resampler) { ret = -ENOMEM; mutex_unlock(&kvm->irqfds.resampler_lock); @@ -797,7 +798,7 @@ static int kvm_assign_ioeventfd_idx(struct kvm *kvm, if (IS_ERR(eventfd)) return PTR_ERR(eventfd); - p = kzalloc(sizeof(*p), GFP_KERNEL); + p = kzalloc(sizeof(*p), GFP_KERNEL_ACCOUNT); if (!p) { ret = -ENOMEM; goto fail; diff --git a/virt/kvm/irqchip.c b/virt/kvm/irqchip.c index b1286c4e07122..3547b0d8c91ea 100644 --- a/virt/kvm/irqchip.c +++ b/virt/kvm/irqchip.c @@ -196,7 +196,7 @@ int kvm_set_irq_routing(struct kvm *kvm, nr_rt_entries += 1; new = kzalloc(sizeof(*new) + (nr_rt_entries * sizeof(struct hlist_head)), - GFP_KERNEL); + GFP_KERNEL_ACCOUNT); if (!new) return -ENOMEM; @@ -208,7 +208,7 @@ int kvm_set_irq_routing(struct kvm *kvm, for (i = 0; i < nr; ++i) { r = -ENOMEM; - e = kzalloc(sizeof(*e), GFP_KERNEL); + e = kzalloc(sizeof(*e), GFP_KERNEL_ACCOUNT); if (!e) goto out; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index d237d3350a99c..f25aa98a94df4 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -81,6 +81,11 @@ unsigned int halt_poll_ns_grow = 2; module_param(halt_poll_ns_grow, uint, 0644); EXPORT_SYMBOL_GPL(halt_poll_ns_grow); +/* The start value to grow halt_poll_ns from */ +unsigned int halt_poll_ns_grow_start = 10000; /* 10us */ +module_param(halt_poll_ns_grow_start, uint, 0644); +EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start); + /* Default resets per-vcpu halt_poll_ns . */ unsigned int halt_poll_ns_shrink; module_param(halt_poll_ns_shrink, uint, 0644); @@ -525,7 +530,7 @@ static struct kvm_memslots *kvm_alloc_memslots(void) int i; struct kvm_memslots *slots; - slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); + slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT); if (!slots) return NULL; @@ -601,12 +606,12 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, int fd) kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries, sizeof(*kvm->debugfs_stat_data), - GFP_KERNEL); + GFP_KERNEL_ACCOUNT); if (!kvm->debugfs_stat_data) return -ENOMEM; for (p = debugfs_entries; p->name; p++) { - stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL); + stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT); if (!stat_data) return -ENOMEM; @@ -656,12 +661,8 @@ static struct kvm *kvm_create_vm(unsigned long type) struct kvm_memslots *slots = kvm_alloc_memslots(); if (!slots) goto out_err_no_srcu; - /* - * Generations must be different for each address space. - * Init kvm generation close to the maximum to easily test the - * code of handling generation number wrap-around. - */ - slots->generation = i * 2 - 150; + /* Generations must be different for each address space. */ + slots->generation = i; rcu_assign_pointer(kvm->memslots[i], slots); } @@ -671,7 +672,7 @@ static struct kvm *kvm_create_vm(unsigned long type) goto out_err_no_irq_srcu; for (i = 0; i < KVM_NR_BUSES; i++) { rcu_assign_pointer(kvm->buses[i], - kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL)); + kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT)); if (!kvm->buses[i]) goto out_err; } @@ -789,7 +790,7 @@ static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) { unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); - memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL); + memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT); if (!memslot->dirty_bitmap) return -ENOMEM; @@ -874,31 +875,34 @@ static struct kvm_memslots *install_new_memslots(struct kvm *kvm, int as_id, struct kvm_memslots *slots) { struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id); + u64 gen = old_memslots->generation; - /* - * Set the low bit in the generation, which disables SPTE caching - * until the end of synchronize_srcu_expedited. - */ - WARN_ON(old_memslots->generation & 1); - slots->generation = old_memslots->generation + 1; + WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS); + slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS; rcu_assign_pointer(kvm->memslots[as_id], slots); synchronize_srcu_expedited(&kvm->srcu); /* - * Increment the new memslot generation a second time. This prevents - * vm exits that race with memslot updates from caching a memslot - * generation that will (potentially) be valid forever. - * + * Increment the new memslot generation a second time, dropping the + * update in-progress flag and incrementing then generation based on + * the number of address spaces. This provides a unique and easily + * identifiable generation number while the memslots are in flux. + */ + gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS; + + /* * Generations must be unique even across address spaces. We do not need * a global counter for that, instead the generation space is evenly split * across address spaces. For example, with two address spaces, address - * space 0 will use generations 0, 4, 8, ... while * address space 1 will - * use generations 2, 6, 10, 14, ... + * space 0 will use generations 0, 2, 4, ... while address space 1 will + * use generations 1, 3, 5, ... */ - slots->generation += KVM_ADDRESS_SPACE_NUM * 2 - 1; + gen += KVM_ADDRESS_SPACE_NUM; + + kvm_arch_memslots_updated(kvm, gen); - kvm_arch_memslots_updated(kvm, slots); + slots->generation = gen; return old_memslots; } @@ -1018,7 +1022,7 @@ int __kvm_set_memory_region(struct kvm *kvm, goto out_free; } - slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); + slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT); if (!slots) goto out_free; memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots)); @@ -1201,11 +1205,9 @@ int kvm_get_dirty_log_protect(struct kvm *kvm, mask = xchg(&dirty_bitmap[i], 0); dirty_bitmap_buffer[i] = mask; - if (mask) { - offset = i * BITS_PER_LONG; - kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, - offset, mask); - } + offset = i * BITS_PER_LONG; + kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, + offset, mask); } spin_unlock(&kvm->mmu_lock); } @@ -2185,20 +2187,23 @@ void kvm_sigset_deactivate(struct kvm_vcpu *vcpu) static void grow_halt_poll_ns(struct kvm_vcpu *vcpu) { - unsigned int old, val, grow; + unsigned int old, val, grow, grow_start; old = val = vcpu->halt_poll_ns; + grow_start = READ_ONCE(halt_poll_ns_grow_start); grow = READ_ONCE(halt_poll_ns_grow); - /* 10us base */ - if (val == 0 && grow) - val = 10000; - else - val *= grow; + if (!grow) + goto out; + + val *= grow; + if (val < grow_start) + val = grow_start; if (val > halt_poll_ns) val = halt_poll_ns; vcpu->halt_poll_ns = val; +out: trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old); } @@ -2683,7 +2688,7 @@ static long kvm_vcpu_ioctl(struct file *filp, struct kvm_regs *kvm_regs; r = -ENOMEM; - kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL); + kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL_ACCOUNT); if (!kvm_regs) goto out; r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs); @@ -2711,7 +2716,8 @@ out_free1: break; } case KVM_GET_SREGS: { - kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL); + kvm_sregs = kzalloc(sizeof(struct kvm_sregs), + GFP_KERNEL_ACCOUNT); r = -ENOMEM; if (!kvm_sregs) goto out; @@ -2803,7 +2809,7 @@ out_free1: break; } case KVM_GET_FPU: { - fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL); + fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL_ACCOUNT); r = -ENOMEM; if (!fpu) goto out; @@ -2980,7 +2986,7 @@ static int kvm_ioctl_create_device(struct kvm *kvm, if (test) return 0; - dev = kzalloc(sizeof(*dev), GFP_KERNEL); + dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT); if (!dev) return -ENOMEM; @@ -3625,6 +3631,7 @@ int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, r = __kvm_io_bus_write(vcpu, bus, &range, val); return r < 0 ? r : 0; } +EXPORT_SYMBOL_GPL(kvm_io_bus_write); /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, @@ -3675,7 +3682,6 @@ static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus, return -EOPNOTSUPP; } -EXPORT_SYMBOL_GPL(kvm_io_bus_write); /* kvm_io_bus_read - called under kvm->slots_lock */ int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, @@ -3697,7 +3703,6 @@ int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, return r < 0 ? r : 0; } - /* Caller must hold slots_lock. */ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len, struct kvm_io_device *dev) @@ -3714,8 +3719,8 @@ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1) return -ENOSPC; - new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) * - sizeof(struct kvm_io_range)), GFP_KERNEL); + new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1), + GFP_KERNEL_ACCOUNT); if (!new_bus) return -ENOMEM; @@ -3760,8 +3765,8 @@ void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, if (i == bus->dev_count) return; - new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) * - sizeof(struct kvm_io_range)), GFP_KERNEL); + new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1), + GFP_KERNEL_ACCOUNT); if (!new_bus) { pr_err("kvm: failed to shrink bus, removing it completely\n"); goto broken; @@ -4029,7 +4034,7 @@ static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) active = kvm_active_vms; spin_unlock(&kvm_lock); - env = kzalloc(sizeof(*env), GFP_KERNEL); + env = kzalloc(sizeof(*env), GFP_KERNEL_ACCOUNT); if (!env) return; @@ -4045,7 +4050,7 @@ static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) add_uevent_var(env, "PID=%d", kvm->userspace_pid); if (!IS_ERR_OR_NULL(kvm->debugfs_dentry)) { - char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL); + char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT); if (p) { tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX); diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c index d99850c462a18..524cbd20379fb 100644 --- a/virt/kvm/vfio.c +++ b/virt/kvm/vfio.c @@ -219,7 +219,7 @@ static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg) } } - kvg = kzalloc(sizeof(*kvg), GFP_KERNEL); + kvg = kzalloc(sizeof(*kvg), GFP_KERNEL_ACCOUNT); if (!kvg) { mutex_unlock(&kv->lock); kvm_vfio_group_put_external_user(vfio_group); @@ -405,7 +405,7 @@ static int kvm_vfio_create(struct kvm_device *dev, u32 type) if (tmp->ops == &kvm_vfio_ops) return -EBUSY; - kv = kzalloc(sizeof(*kv), GFP_KERNEL); + kv = kzalloc(sizeof(*kv), GFP_KERNEL_ACCOUNT); if (!kv) return -ENOMEM; |