korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-12-04 09:34:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

accel/ivpu: Split IP and buttress code

Browse Source 
The NPU device consists of two parts: NPU buttress and NPU IP.
Buttress is a platform specific part that integrates the NPU IP with
the CPU.
NPU IP is the platform agnostic part that does the inference.

This separation enables support for multiple platforms using
a single NPU IP, so for example NPU IP 37XX could be integrated into
MTL and LNL platforms.

Signed-off-by: Wachowski, Karol <karol.wachowski@intel.com>
Signed-off-by: Jacek Lawrynowicz <jacek.lawrynowicz@linux.intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20240515113006.457472-3-jacek.lawrynowicz@linux.intel.com
This commit is contained in:
Wachowski, Karol 2024-05-15 13:30:05 +02:00 committed by Jacek Lawrynowicz
parent 302d583211
commit 8a27ad81f7
15 changed files with 2588 additions and 2502 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
drivers/accel/ivpu/Makefile
View File

@ -6,8 +6,9 @@ intel_vpu-y := \
ivpu_fw.o \ ivpu_fw.o \
ivpu_fw_log.o \ ivpu_fw_log.o \
ivpu_gem.o \ ivpu_gem.o \
ivpu_hw_37xx.o \ ivpu_hw.o \
ivpu_hw_40xx.o \ ivpu_hw_btrs.o \
ivpu_hw_ip.o \
ivpu_ipc.o \ ivpu_ipc.o \
ivpu_job.o \ ivpu_job.o \
ivpu_jsm_msg.o \ ivpu_jsm_msg.o \

2
drivers/accel/ivpu/ivpu_debugfs.c
View File

@ -409,7 +409,7 @@ void ivpu_debugfs_init(struct ivpu_device *vdev)
debugfs_create_file("resume_engine", 0200, debugfs_root, vdev, debugfs_create_file("resume_engine", 0200, debugfs_root, vdev,
&ivpu_resume_engine_fops); &ivpu_resume_engine_fops);
if (ivpu_hw_gen(vdev) >= IVPU_HW_40XX) if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX)
debugfs_create_file("fw_profiling_freq_drive", 0200, debugfs_create_file("fw_profiling_freq_drive", 0200,
debugfs_root, vdev, &fw_profiling_freq_fops); debugfs_root, vdev, &fw_profiling_freq_fops);
} }

13
drivers/accel/ivpu/ivpu_drv.c
View File

@ -464,9 +464,11 @@ static int ivpu_irq_init(struct ivpu_device *vdev)
return ret; return ret;
} }
ivpu_irq_handlers_init(vdev);
vdev->irq = pci_irq_vector(pdev, 0); vdev->irq = pci_irq_vector(pdev, 0);
ret = devm_request_threaded_irq(vdev->drm.dev, vdev->irq, vdev->hw->ops->irq_handler, ret = devm_request_threaded_irq(vdev->drm.dev, vdev->irq, ivpu_hw_irq_handler,
ivpu_irq_thread_handler, IRQF_NO_AUTOEN, DRIVER_NAME, vdev); ivpu_irq_thread_handler, IRQF_NO_AUTOEN, DRIVER_NAME, vdev);
if (ret) if (ret)
ivpu_err(vdev, "Failed to request an IRQ %d\n", ret); ivpu_err(vdev, "Failed to request an IRQ %d\n", ret);
@ -543,13 +545,10 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
if (!vdev->pm) if (!vdev->pm)
return -ENOMEM; return -ENOMEM;
if (ivpu_hw_gen(vdev) >= IVPU_HW_40XX) { if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX)
vdev->hw->ops = &ivpu_hw_40xx_ops;
vdev->hw->dma_bits = 48; vdev->hw->dma_bits = 48;
} else { else
vdev->hw->ops = &ivpu_hw_37xx_ops;
vdev->hw->dma_bits = 38; vdev->hw->dma_bits = 38;
}
vdev->platform = IVPU_PLATFORM_INVALID; vdev->platform = IVPU_PLATFORM_INVALID;
vdev->context_xa_limit.min = IVPU_USER_CONTEXT_MIN_SSID; vdev->context_xa_limit.min = IVPU_USER_CONTEXT_MIN_SSID;
@ -578,7 +577,7 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
goto err_xa_destroy; goto err_xa_destroy;
/* Init basic HW info based on buttress registers which are accessible before power up */ /* Init basic HW info based on buttress registers which are accessible before power up */
ret = ivpu_hw_info_init(vdev); ret = ivpu_hw_init(vdev);
if (ret) if (ret)
goto err_xa_destroy; goto err_xa_destroy;

33
drivers/accel/ivpu/ivpu_drv.h
View File

@ -27,8 +27,13 @@
#define PCI_DEVICE_ID_ARL 0xad1d #define PCI_DEVICE_ID_ARL 0xad1d
#define PCI_DEVICE_ID_LNL 0x643e #define PCI_DEVICE_ID_LNL 0x643e
#define IVPU_HW_37XX 37 #define IVPU_HW_IP_37XX 37
#define IVPU_HW_40XX 40 #define IVPU_HW_IP_40XX 40
#define IVPU_HW_IP_50XX 50
#define IVPU_HW_IP_60XX 60
#define IVPU_HW_BTRS_MTL 1
#define IVPU_HW_BTRS_LNL 2
#define IVPU_GLOBAL_CONTEXT_MMU_SSID 0 #define IVPU_GLOBAL_CONTEXT_MMU_SSID 0
/* SSID 1 is used by the VPU to represent reserved context */ /* SSID 1 is used by the VPU to represent reserved context */
@ -198,16 +203,32 @@ static inline u16 ivpu_device_id(struct ivpu_device *vdev)
return to_pci_dev(vdev->drm.dev)->device; return to_pci_dev(vdev->drm.dev)->device;
} }
static inline int ivpu_hw_gen(struct ivpu_device *vdev) static inline int ivpu_hw_ip_gen(struct ivpu_device *vdev)
{ {
switch (ivpu_device_id(vdev)) { switch (ivpu_device_id(vdev)) {
case PCI_DEVICE_ID_MTL: case PCI_DEVICE_ID_MTL:
case PCI_DEVICE_ID_ARL: case PCI_DEVICE_ID_ARL:
return IVPU_HW_37XX; return IVPU_HW_IP_37XX;
case PCI_DEVICE_ID_LNL: case PCI_DEVICE_ID_LNL:
return IVPU_HW_40XX; return IVPU_HW_IP_40XX;
default: default:
ivpu_err(vdev, "Unknown NPU device\n"); dump_stack();
ivpu_err(vdev, "Unknown NPU IP generation\n");
return 0;
}
}
static inline int ivpu_hw_btrs_gen(struct ivpu_device *vdev)
{
switch (ivpu_device_id(vdev)) {
case PCI_DEVICE_ID_MTL:
case PCI_DEVICE_ID_ARL:
return IVPU_HW_BTRS_MTL;
case PCI_DEVICE_ID_LNL:
return IVPU_HW_BTRS_LNL;
default:
dump_stack();
ivpu_err(vdev, "Unknown buttress generation\n");
return 0; return 0;
} }
} }

20
drivers/accel/ivpu/ivpu_fw.c
View File

@ -54,10 +54,10 @@ static struct {
int gen; int gen;
const char *name; const char *name;
} fw_names[] = { } fw_names[] = {
{ IVPU_HW_37XX, "vpu_37xx.bin" }, { IVPU_HW_IP_37XX, "vpu_37xx.bin" },
{ IVPU_HW_37XX, "intel/vpu/vpu_37xx_v0.0.bin" }, { IVPU_HW_IP_37XX, "intel/vpu/vpu_37xx_v0.0.bin" },
{ IVPU_HW_40XX, "vpu_40xx.bin" }, { IVPU_HW_IP_40XX, "vpu_40xx.bin" },
{ IVPU_HW_40XX, "intel/vpu/vpu_40xx_v0.0.bin" }, { IVPU_HW_IP_40XX, "intel/vpu/vpu_40xx_v0.0.bin" },
}; };
static int ivpu_fw_request(struct ivpu_device *vdev) static int ivpu_fw_request(struct ivpu_device *vdev)
@ -73,7 +73,7 @@ static int ivpu_fw_request(struct ivpu_device *vdev)
} }
for (i = 0; i < ARRAY_SIZE(fw_names); i++) { for (i = 0; i < ARRAY_SIZE(fw_names); i++) {
if (fw_names[i].gen != ivpu_hw_gen(vdev)) if (fw_names[i].gen != ivpu_hw_ip_gen(vdev))
continue; continue;
ret = firmware_request_nowarn(&vdev->fw->file, fw_names[i].name, vdev->drm.dev); ret = firmware_request_nowarn(&vdev->fw->file, fw_names[i].name, vdev->drm.dev);
@ -246,7 +246,7 @@ static int ivpu_fw_update_global_range(struct ivpu_device *vdev)
return -EINVAL; return -EINVAL;
} }
ivpu_hw_init_range(&vdev->hw->ranges.global, start, size); ivpu_hw_range_init(&vdev->hw->ranges.global, start, size);
return 0; return 0;
} }
@ -511,7 +511,7 @@ void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params
boot_params->magic = VPU_BOOT_PARAMS_MAGIC; boot_params->magic = VPU_BOOT_PARAMS_MAGIC;
boot_params->vpu_id = to_pci_dev(vdev->drm.dev)->bus->number; boot_params->vpu_id = to_pci_dev(vdev->drm.dev)->bus->number;
boot_params->frequency = ivpu_hw_reg_pll_freq_get(vdev); boot_params->frequency = ivpu_hw_pll_freq_get(vdev);
/* /*
* This param is a debug firmware feature. It switches default clock * This param is a debug firmware feature. It switches default clock
@ -568,9 +568,9 @@ void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params
boot_params->verbose_tracing_buff_addr = vdev->fw->mem_log_verb->vpu_addr; boot_params->verbose_tracing_buff_addr = vdev->fw->mem_log_verb->vpu_addr;
boot_params->verbose_tracing_buff_size = ivpu_bo_size(vdev->fw->mem_log_verb); boot_params->verbose_tracing_buff_size = ivpu_bo_size(vdev->fw->mem_log_verb);
boot_params->punit_telemetry_sram_base = ivpu_hw_reg_telemetry_offset_get(vdev); boot_params->punit_telemetry_sram_base = ivpu_hw_telemetry_offset_get(vdev);
boot_params->punit_telemetry_sram_size = ivpu_hw_reg_telemetry_size_get(vdev); boot_params->punit_telemetry_sram_size = ivpu_hw_telemetry_size_get(vdev);
boot_params->vpu_telemetry_enable = ivpu_hw_reg_telemetry_enable_get(vdev); boot_params->vpu_telemetry_enable = ivpu_hw_telemetry_enable_get(vdev);
boot_params->vpu_scheduling_mode = vdev->hw->sched_mode; boot_params->vpu_scheduling_mode = vdev->hw->sched_mode;
if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW) if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW)
boot_params->vpu_focus_present_timer_ms = IVPU_FOCUS_PRESENT_TIMER_MS; boot_params->vpu_focus_present_timer_ms = IVPU_FOCUS_PRESENT_TIMER_MS;

310
drivers/accel/ivpu/ivpu_hw.c Normal file
View File

@ -0,0 +1,310 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2020 - 2024 Intel Corporation
*/
#include "ivpu_drv.h"
#include "ivpu_hw.h"
#include "ivpu_hw_btrs.h"
#include "ivpu_hw_ip.h"
#include <linux/dmi.h>
static char *platform_to_str(u32 platform)
{
switch (platform) {
case IVPU_PLATFORM_SILICON:
return "SILICON";
case IVPU_PLATFORM_SIMICS:
return "SIMICS";
case IVPU_PLATFORM_FPGA:
return "FPGA";
default:
return "Invalid platform";
}
}
static const struct dmi_system_id dmi_platform_simulation[] = {
{
.ident = "Intel Simics",
.matches = {
DMI_MATCH(DMI_BOARD_NAME, "lnlrvp"),
DMI_MATCH(DMI_BOARD_VERSION, "1.0"),
DMI_MATCH(DMI_BOARD_SERIAL, "123456789"),
},
},
{
.ident = "Intel Simics",
.matches = {
DMI_MATCH(DMI_BOARD_NAME, "Simics"),
},
},
{ }
};
static void platform_init(struct ivpu_device *vdev)
{
if (dmi_check_system(dmi_platform_simulation))
vdev->platform = IVPU_PLATFORM_SIMICS;
else
vdev->platform = IVPU_PLATFORM_SILICON;
ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n",
platform_to_str(vdev->platform), vdev->platform);
}
static void wa_init(struct ivpu_device *vdev)
{
vdev->wa.punit_disabled = ivpu_is_fpga(vdev);
vdev->wa.clear_runtime_mem = false;
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
vdev->wa.interrupt_clear_with_0 = ivpu_hw_btrs_irqs_clear_with_0_mtl(vdev);
if (ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL)
vdev->wa.disable_clock_relinquish = true;
IVPU_PRINT_WA(punit_disabled);
IVPU_PRINT_WA(clear_runtime_mem);
IVPU_PRINT_WA(interrupt_clear_with_0);
IVPU_PRINT_WA(disable_clock_relinquish);
}
static void timeouts_init(struct ivpu_device *vdev)
{
if (ivpu_is_fpga(vdev)) {
vdev->timeout.boot = 100000;
vdev->timeout.jsm = 50000;
vdev->timeout.tdr = 2000000;
vdev->timeout.reschedule_suspend = 1000;
vdev->timeout.autosuspend = -1;
vdev->timeout.d0i3_entry_msg = 500;
} else if (ivpu_is_simics(vdev)) {
vdev->timeout.boot = 50;
vdev->timeout.jsm = 500;
vdev->timeout.tdr = 10000;
vdev->timeout.reschedule_suspend = 10;
vdev->timeout.autosuspend = -1;
vdev->timeout.d0i3_entry_msg = 100;
} else {
vdev->timeout.boot = 1000;
vdev->timeout.jsm = 500;
vdev->timeout.tdr = 2000;
vdev->timeout.reschedule_suspend = 10;
vdev->timeout.autosuspend = 10;
vdev->timeout.d0i3_entry_msg = 5;
}
}
static void memory_ranges_init(struct ivpu_device *vdev)
{
if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
ivpu_hw_range_init(&vdev->hw->ranges.global, 0x80000000, SZ_512M);
ivpu_hw_range_init(&vdev->hw->ranges.user, 0xc0000000, 255 * SZ_1M);
ivpu_hw_range_init(&vdev->hw->ranges.shave, 0x180000000, SZ_2G);
ivpu_hw_range_init(&vdev->hw->ranges.dma, 0x200000000, SZ_8G);
} else {
ivpu_hw_range_init(&vdev->hw->ranges.global, 0x80000000, SZ_512M);
ivpu_hw_range_init(&vdev->hw->ranges.user, 0x80000000, SZ_256M);
ivpu_hw_range_init(&vdev->hw->ranges.shave, 0x80000000 + SZ_256M, SZ_2G - SZ_256M);
ivpu_hw_range_init(&vdev->hw->ranges.dma, 0x200000000, SZ_8G);
}
}
static int wp_enable(struct ivpu_device *vdev)
{
return ivpu_hw_btrs_wp_drive(vdev, true);
}
static int wp_disable(struct ivpu_device *vdev)
{
return ivpu_hw_btrs_wp_drive(vdev, false);
}
int ivpu_hw_power_up(struct ivpu_device *vdev)
{
int ret;
ret = ivpu_hw_btrs_d0i3_disable(vdev);
if (ret)
ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
ret = wp_enable(vdev);
if (ret) {
ivpu_err(vdev, "Failed to enable workpoint: %d\n", ret);
return ret;
}
if (ivpu_hw_btrs_gen(vdev) >= IVPU_HW_BTRS_LNL) {
if (IVPU_WA(disable_clock_relinquish))
ivpu_hw_btrs_clock_relinquish_disable_lnl(vdev);
ivpu_hw_btrs_profiling_freq_reg_set_lnl(vdev);
ivpu_hw_btrs_ats_print_lnl(vdev);
}
ret = ivpu_hw_ip_host_ss_configure(vdev);
if (ret) {
ivpu_err(vdev, "Failed to configure host SS: %d\n", ret);
return ret;
}
ivpu_hw_ip_idle_gen_disable(vdev);
ret = ivpu_hw_btrs_wait_for_clock_res_own_ack(vdev);
if (ret) {
ivpu_err(vdev, "Timed out waiting for clock resource own ACK\n");
return ret;
}
ret = ivpu_hw_ip_pwr_domain_enable(vdev);
if (ret) {
ivpu_err(vdev, "Failed to enable power domain: %d\n", ret);
return ret;
}
ret = ivpu_hw_ip_host_ss_axi_enable(vdev);
if (ret) {
ivpu_err(vdev, "Failed to enable AXI: %d\n", ret);
return ret;
}
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_LNL)
ivpu_hw_btrs_set_port_arbitration_weights_lnl(vdev);
ret = ivpu_hw_ip_top_noc_enable(vdev);
if (ret)
ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret);
return ret;
}
static void save_d0i3_entry_timestamp(struct ivpu_device *vdev)
{
vdev->hw->d0i3_entry_host_ts = ktime_get_boottime();
vdev->hw->d0i3_entry_vpu_ts = ivpu_hw_ip_read_perf_timer_counter(vdev);
}
int ivpu_hw_reset(struct ivpu_device *vdev)
{
int ret = 0;
if (ivpu_hw_btrs_ip_reset(vdev)) {
ivpu_err(vdev, "Failed to reset NPU IP\n");
ret = -EIO;
}
if (wp_disable(vdev)) {
ivpu_err(vdev, "Failed to disable workpoint\n");
ret = -EIO;
}
return ret;
}
int ivpu_hw_power_down(struct ivpu_device *vdev)
{
int ret = 0;
save_d0i3_entry_timestamp(vdev);
if (!ivpu_hw_is_idle(vdev))
ivpu_warn(vdev, "NPU not idle during power down\n");
if (ivpu_hw_reset(vdev)) {
ivpu_err(vdev, "Failed to reset NPU\n");
ret = -EIO;
}
if (ivpu_hw_btrs_d0i3_enable(vdev)) {
ivpu_err(vdev, "Failed to enter D0I3\n");
ret = -EIO;
}
return ret;
}
int ivpu_hw_init(struct ivpu_device *vdev)
{
ivpu_hw_btrs_info_init(vdev);
ivpu_hw_btrs_freq_ratios_init(vdev);
memory_ranges_init(vdev);
platform_init(vdev);
wa_init(vdev);
timeouts_init(vdev);
return 0;
}
int ivpu_hw_boot_fw(struct ivpu_device *vdev)
{
int ret;
ivpu_hw_ip_snoop_disable(vdev);
ivpu_hw_ip_tbu_mmu_enable(vdev);
ret = ivpu_hw_ip_soc_cpu_boot(vdev);
if (ret)
ivpu_err(vdev, "Failed to boot SOC CPU: %d\n", ret);
return ret;
}
void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
{
if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
return;
}
if (enable)
vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_HIGH;
else
vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
}
void ivpu_irq_handlers_init(struct ivpu_device *vdev)
{
if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_37xx;
else
vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_40xx;
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_mtl;
else
vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_lnl;
}
void ivpu_hw_irq_enable(struct ivpu_device *vdev)
{
ivpu_hw_ip_irq_enable(vdev);
ivpu_hw_btrs_irq_enable(vdev);
}
void ivpu_hw_irq_disable(struct ivpu_device *vdev)
{
ivpu_hw_btrs_irq_disable(vdev);
ivpu_hw_ip_irq_disable(vdev);
}
irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr)
{
bool ip_handled, btrs_handled, wake_thread = false;
struct ivpu_device *vdev = ptr;
ivpu_hw_btrs_global_int_disable(vdev);
btrs_handled = ivpu_hw_btrs_irq_handler(vdev, irq);
if (!ivpu_hw_is_idle((vdev)) || !btrs_handled)
ip_handled = ivpu_hw_ip_irq_handler(vdev, irq, &wake_thread);
else
ip_handled = false;
/* Re-enable global interrupts to re-trigger MSI for pending interrupts */
ivpu_hw_btrs_global_int_enable(vdev);
if (wake_thread)
return IRQ_WAKE_THREAD;
if (ip_handled || btrs_handled)
return IRQ_HANDLED;
return IRQ_NONE;
}

235
drivers/accel/ivpu/ivpu_hw.h
View File

@ -1,39 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0-only */ /* SPDX-License-Identifier: GPL-2.0-only */
/* /*
* Copyright (C) 2020-2024 Intel Corporation * Copyright (C) 2020 - 2024 Intel Corporation
*/ */
#ifndef __IVPU_HW_H__ #ifndef __IVPU_HW_H__
#define __IVPU_HW_H__ #define __IVPU_HW_H__
#include "ivpu_drv.h" #include "ivpu_drv.h"
#include "ivpu_hw_btrs.h"
struct ivpu_hw_ops { #include "ivpu_hw_ip.h"
int (*info_init)(struct ivpu_device *vdev);
int (*power_up)(struct ivpu_device *vdev);
int (*boot_fw)(struct ivpu_device *vdev);
int (*power_down)(struct ivpu_device *vdev);
int (*reset)(struct ivpu_device *vdev);
bool (*is_idle)(struct ivpu_device *vdev);
int (*wait_for_idle)(struct ivpu_device *vdev);
void (*wdt_disable)(struct ivpu_device *vdev);
void (*diagnose_failure)(struct ivpu_device *vdev);
u32 (*profiling_freq_get)(struct ivpu_device *vdev);
void (*profiling_freq_drive)(struct ivpu_device *vdev, bool enable);
u32 (*reg_pll_freq_get)(struct ivpu_device *vdev);
u32 (*ratio_to_freq)(struct ivpu_device *vdev, u32 ratio);
u32 (*reg_telemetry_offset_get)(struct ivpu_device *vdev);
u32 (*reg_telemetry_size_get)(struct ivpu_device *vdev);
u32 (*reg_telemetry_enable_get)(struct ivpu_device *vdev);
void (*reg_db_set)(struct ivpu_device *vdev, u32 db_id);
u32 (*reg_ipc_rx_addr_get)(struct ivpu_device *vdev);
u32 (*reg_ipc_rx_count_get)(struct ivpu_device *vdev);
void (*reg_ipc_tx_set)(struct ivpu_device *vdev, u32 vpu_addr);
void (*irq_clear)(struct ivpu_device *vdev);
void (*irq_enable)(struct ivpu_device *vdev);
void (*irq_disable)(struct ivpu_device *vdev);
irqreturn_t (*irq_handler)(int irq, void *ptr);
};
struct ivpu_addr_range { struct ivpu_addr_range {
resource_size_t start; resource_size_t start;
@ -41,7 +16,10 @@ struct ivpu_addr_range {
}; };
struct ivpu_hw_info { struct ivpu_hw_info {
const struct ivpu_hw_ops *ops; struct {
bool (*btrs_irq_handler)(struct ivpu_device *vdev, int irq);
bool (*ip_irq_handler)(struct ivpu_device *vdev, int irq, bool *wake_thread);
} irq;
struct { struct {
struct ivpu_addr_range global; struct ivpu_addr_range global;
struct ivpu_addr_range user; struct ivpu_addr_range user;
@ -67,127 +45,28 @@ struct ivpu_hw_info {
u64 d0i3_entry_vpu_ts; u64 d0i3_entry_vpu_ts;
}; };
extern const struct ivpu_hw_ops ivpu_hw_37xx_ops; int ivpu_hw_init(struct ivpu_device *vdev);
extern const struct ivpu_hw_ops ivpu_hw_40xx_ops; int ivpu_hw_power_up(struct ivpu_device *vdev);
int ivpu_hw_power_down(struct ivpu_device *vdev);
int ivpu_hw_reset(struct ivpu_device *vdev);
int ivpu_hw_boot_fw(struct ivpu_device *vdev);
void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable);
void ivpu_irq_handlers_init(struct ivpu_device *vdev);
void ivpu_hw_irq_enable(struct ivpu_device *vdev);
void ivpu_hw_irq_disable(struct ivpu_device *vdev);
irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr);
static inline int ivpu_hw_info_init(struct ivpu_device *vdev) static inline u32 ivpu_hw_btrs_irq_handler(struct ivpu_device *vdev, int irq)
{ {
return vdev->hw->ops->info_init(vdev); return vdev->hw->irq.btrs_irq_handler(vdev, irq);
};
static inline int ivpu_hw_power_up(struct ivpu_device *vdev)
{
ivpu_dbg(vdev, PM, "HW power up\n");
return vdev->hw->ops->power_up(vdev);
};
static inline int ivpu_hw_boot_fw(struct ivpu_device *vdev)
{
return vdev->hw->ops->boot_fw(vdev);
};
static inline bool ivpu_hw_is_idle(struct ivpu_device *vdev)
{
return vdev->hw->ops->is_idle(vdev);
};
static inline int ivpu_hw_wait_for_idle(struct ivpu_device *vdev)
{
return vdev->hw->ops->wait_for_idle(vdev);
};
static inline int ivpu_hw_power_down(struct ivpu_device *vdev)
{
ivpu_dbg(vdev, PM, "HW power down\n");
return vdev->hw->ops->power_down(vdev);
};
static inline int ivpu_hw_reset(struct ivpu_device *vdev)
{
ivpu_dbg(vdev, PM, "HW reset\n");
return vdev->hw->ops->reset(vdev);
};
static inline void ivpu_hw_wdt_disable(struct ivpu_device *vdev)
{
vdev->hw->ops->wdt_disable(vdev);
};
static inline u32 ivpu_hw_profiling_freq_get(struct ivpu_device *vdev)
{
return vdev->hw->ops->profiling_freq_get(vdev);
};
static inline void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
{
return vdev->hw->ops->profiling_freq_drive(vdev, enable);
};
/* Register indirect accesses */
static inline u32 ivpu_hw_reg_pll_freq_get(struct ivpu_device *vdev)
{
return vdev->hw->ops->reg_pll_freq_get(vdev);
};
static inline u32 ivpu_hw_ratio_to_freq(struct ivpu_device *vdev, u32 ratio)
{
return vdev->hw->ops->ratio_to_freq(vdev, ratio);
} }
static inline u32 ivpu_hw_reg_telemetry_offset_get(struct ivpu_device *vdev) static inline u32 ivpu_hw_ip_irq_handler(struct ivpu_device *vdev, int irq, bool *wake_thread)
{ {
return vdev->hw->ops->reg_telemetry_offset_get(vdev); return vdev->hw->irq.ip_irq_handler(vdev, irq, wake_thread);
}; }
static inline u32 ivpu_hw_reg_telemetry_size_get(struct ivpu_device *vdev) static inline void ivpu_hw_range_init(struct ivpu_addr_range *range, u64 start, u64 size)
{
return vdev->hw->ops->reg_telemetry_size_get(vdev);
};
static inline u32 ivpu_hw_reg_telemetry_enable_get(struct ivpu_device *vdev)
{
return vdev->hw->ops->reg_telemetry_enable_get(vdev);
};
static inline void ivpu_hw_reg_db_set(struct ivpu_device *vdev, u32 db_id)
{
vdev->hw->ops->reg_db_set(vdev, db_id);
};
static inline u32 ivpu_hw_reg_ipc_rx_addr_get(struct ivpu_device *vdev)
{
return vdev->hw->ops->reg_ipc_rx_addr_get(vdev);
};
static inline u32 ivpu_hw_reg_ipc_rx_count_get(struct ivpu_device *vdev)
{
return vdev->hw->ops->reg_ipc_rx_count_get(vdev);
};
static inline void ivpu_hw_reg_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr)
{
vdev->hw->ops->reg_ipc_tx_set(vdev, vpu_addr);
};
static inline void ivpu_hw_irq_clear(struct ivpu_device *vdev)
{
vdev->hw->ops->irq_clear(vdev);
};
static inline void ivpu_hw_irq_enable(struct ivpu_device *vdev)
{
vdev->hw->ops->irq_enable(vdev);
};
static inline void ivpu_hw_irq_disable(struct ivpu_device *vdev)
{
vdev->hw->ops->irq_disable(vdev);
};
static inline void ivpu_hw_init_range(struct ivpu_addr_range *range, u64 start, u64 size)
{ {
range->start = start; range->start = start;
range->end = start + size; range->end = start + size;
@ -198,9 +77,75 @@ static inline u64 ivpu_hw_range_size(const struct ivpu_addr_range *range)
return range->end - range->start; return range->end - range->start;
} }
static inline u32 ivpu_hw_ratio_to_freq(struct ivpu_device *vdev, u32 ratio)
{
return ivpu_hw_btrs_ratio_to_freq(vdev, ratio);
}
static inline void ivpu_hw_irq_clear(struct ivpu_device *vdev)
{
ivpu_hw_ip_irq_clear(vdev);
}
static inline u32 ivpu_hw_pll_freq_get(struct ivpu_device *vdev)
{
return ivpu_hw_btrs_pll_freq_get(vdev);
}
static inline u32 ivpu_hw_profiling_freq_get(struct ivpu_device *vdev)
{
return vdev->hw->pll.profiling_freq;
}
static inline void ivpu_hw_diagnose_failure(struct ivpu_device *vdev) static inline void ivpu_hw_diagnose_failure(struct ivpu_device *vdev)
{ {
vdev->hw->ops->diagnose_failure(vdev); ivpu_hw_ip_diagnose_failure(vdev);
ivpu_hw_btrs_diagnose_failure(vdev);
}
static inline u32 ivpu_hw_telemetry_offset_get(struct ivpu_device *vdev)
{
return ivpu_hw_btrs_telemetry_offset_get(vdev);
}
static inline u32 ivpu_hw_telemetry_size_get(struct ivpu_device *vdev)
{
return ivpu_hw_btrs_telemetry_size_get(vdev);
}
static inline u32 ivpu_hw_telemetry_enable_get(struct ivpu_device *vdev)
{
return ivpu_hw_btrs_telemetry_enable_get(vdev);
}
static inline bool ivpu_hw_is_idle(struct ivpu_device *vdev)
{
return ivpu_hw_btrs_is_idle(vdev);
}
static inline int ivpu_hw_wait_for_idle(struct ivpu_device *vdev)
{
return ivpu_hw_btrs_wait_for_idle(vdev);
}
static inline void ivpu_hw_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr)
{
ivpu_hw_ip_ipc_tx_set(vdev, vpu_addr);
}
static inline void ivpu_hw_db_set(struct ivpu_device *vdev, u32 db_id)
{
ivpu_hw_ip_db_set(vdev, db_id);
}
static inline u32 ivpu_hw_ipc_rx_addr_get(struct ivpu_device *vdev)
{
return ivpu_hw_ip_ipc_rx_addr_get(vdev);
}
static inline u32 ivpu_hw_ipc_rx_count_get(struct ivpu_device *vdev)
{
return ivpu_hw_ip_ipc_rx_count_get(vdev);
} }
#endif /* __IVPU_HW_H__ */ #endif /* __IVPU_HW_H__ */

1071
drivers/accel/ivpu/ivpu_hw_37xx.c
View File

File diff suppressed because it is too large Load Diff

1256
drivers/accel/ivpu/ivpu_hw_40xx.c
View File

File diff suppressed because it is too large Load Diff

881
drivers/accel/ivpu/ivpu_hw_btrs.c Normal file
View File

@ -0,0 +1,881 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2020-2024 Intel Corporation
*/
#include "ivpu_drv.h"
#include "ivpu_hw.h"
#include "ivpu_hw_btrs.h"
#include "ivpu_hw_btrs_lnl_reg.h"
#include "ivpu_hw_btrs_mtl_reg.h"
#include "ivpu_hw_reg_io.h"
#include "ivpu_pm.h"
#define BTRS_MTL_IRQ_MASK ((REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR)) | \
(REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR)))
#define BTRS_LNL_IRQ_MASK ((REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR)) | \
(REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR)) | \
(REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR)) | \
(REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR)) | \
(REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR)) | \
(REG_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR)))
#define BTRS_MTL_ALL_IRQ_MASK (BTRS_MTL_IRQ_MASK | (REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, \
FREQ_CHANGE)))
#define BTRS_IRQ_DISABLE_MASK ((u32)-1)
#define BTRS_LNL_ALL_IRQ_MASK ((u32)-1)
#define BTRS_MTL_WP_CONFIG_1_TILE_5_3_RATIO WP_CONFIG(MTL_CONFIG_1_TILE, MTL_PLL_RATIO_5_3)
#define BTRS_MTL_WP_CONFIG_1_TILE_4_3_RATIO WP_CONFIG(MTL_CONFIG_1_TILE, MTL_PLL_RATIO_4_3)
#define BTRS_MTL_WP_CONFIG_2_TILE_5_3_RATIO WP_CONFIG(MTL_CONFIG_2_TILE, MTL_PLL_RATIO_5_3)
#define BTRS_MTL_WP_CONFIG_2_TILE_4_3_RATIO WP_CONFIG(MTL_CONFIG_2_TILE, MTL_PLL_RATIO_4_3)
#define BTRS_MTL_WP_CONFIG_0_TILE_PLL_OFF WP_CONFIG(0, 0)
#define PLL_CDYN_DEFAULT 0x80
#define PLL_EPP_DEFAULT 0x80
#define PLL_CONFIG_DEFAULT 0x0
#define PLL_SIMULATION_FREQ 10000000
#define PLL_REF_CLK_FREQ 50000000
#define PLL_TIMEOUT_US (1500 * USEC_PER_MSEC)
#define IDLE_TIMEOUT_US (5 * USEC_PER_MSEC)
#define TIMEOUT_US (150 * USEC_PER_MSEC)
/* Work point configuration values */
#define WP_CONFIG(tile, ratio) (((tile) << 8) | (ratio))
#define MTL_CONFIG_1_TILE 0x01
#define MTL_CONFIG_2_TILE 0x02
#define MTL_PLL_RATIO_5_3 0x01
#define MTL_PLL_RATIO_4_3 0x02
#define BTRS_MTL_TILE_FUSE_ENABLE_BOTH 0x0
#define BTRS_MTL_TILE_SKU_BOTH 0x3630
#define BTRS_LNL_TILE_MAX_NUM 6
#define BTRS_LNL_TILE_MAX_MASK 0x3f
#define WEIGHTS_DEFAULT 0xf711f711u
#define WEIGHTS_ATS_DEFAULT 0x0000f711u
#define DCT_REQ 0x2
#define DCT_ENABLE 0x1
#define DCT_DISABLE 0x0
int ivpu_hw_btrs_irqs_clear_with_0_mtl(struct ivpu_device *vdev)
{
REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, BTRS_MTL_ALL_IRQ_MASK);
if (REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) == BTRS_MTL_ALL_IRQ_MASK) {
/* Writing 1s does not clear the interrupt status register */
REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, 0x0);
return true;
}
return false;
}
static void freq_ratios_init_mtl(struct ivpu_device *vdev)
{
struct ivpu_hw_info *hw = vdev->hw;
u32 fmin_fuse, fmax_fuse;
fmin_fuse = REGB_RD32(VPU_HW_BTRS_MTL_FMIN_FUSE);
hw->pll.min_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMIN_FUSE, MIN_RATIO, fmin_fuse);
hw->pll.pn_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMIN_FUSE, PN_RATIO, fmin_fuse);
fmax_fuse = REGB_RD32(VPU_HW_BTRS_MTL_FMAX_FUSE);
hw->pll.max_ratio = REG_GET_FLD(VPU_HW_BTRS_MTL_FMAX_FUSE, MAX_RATIO, fmax_fuse);
}
static void freq_ratios_init_lnl(struct ivpu_device *vdev)
{
struct ivpu_hw_info *hw = vdev->hw;
u32 fmin_fuse, fmax_fuse;
fmin_fuse = REGB_RD32(VPU_HW_BTRS_LNL_FMIN_FUSE);
hw->pll.min_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMIN_FUSE, MIN_RATIO, fmin_fuse);
hw->pll.pn_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMIN_FUSE, PN_RATIO, fmin_fuse);
fmax_fuse = REGB_RD32(VPU_HW_BTRS_LNL_FMAX_FUSE);
hw->pll.max_ratio = REG_GET_FLD(VPU_HW_BTRS_LNL_FMAX_FUSE, MAX_RATIO, fmax_fuse);
}
void ivpu_hw_btrs_freq_ratios_init(struct ivpu_device *vdev)
{
struct ivpu_hw_info *hw = vdev->hw;
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
freq_ratios_init_mtl(vdev);
else
freq_ratios_init_lnl(vdev);
hw->pll.min_ratio = clamp_t(u8, ivpu_pll_min_ratio, hw->pll.min_ratio, hw->pll.max_ratio);
hw->pll.max_ratio = clamp_t(u8, ivpu_pll_max_ratio, hw->pll.min_ratio, hw->pll.max_ratio);
hw->pll.pn_ratio = clamp_t(u8, hw->pll.pn_ratio, hw->pll.min_ratio, hw->pll.max_ratio);
}
static bool tile_disable_check(u32 config)
{
/* Allowed values: 0 or one bit from range 0-5 (6 tiles) */
if (config == 0)
return true;
if (config > BIT(BTRS_LNL_TILE_MAX_NUM - 1))
return false;
if ((config & (config - 1)) == 0)
return true;
return false;
}
static int read_tile_config_fuse(struct ivpu_device *vdev, u32 *tile_fuse_config)
{
u32 fuse;
u32 config;
fuse = REGB_RD32(VPU_HW_BTRS_LNL_TILE_FUSE);
if (!REG_TEST_FLD(VPU_HW_BTRS_LNL_TILE_FUSE, VALID, fuse)) {
ivpu_err(vdev, "Fuse: invalid (0x%x)\n", fuse);
return -EIO;
}
config = REG_GET_FLD(VPU_HW_BTRS_LNL_TILE_FUSE, CONFIG, fuse);
if (!tile_disable_check(config)) {
ivpu_err(vdev, "Fuse: Invalid tile disable config (0x%x)\n", config);
return -EIO;
}
if (config)
ivpu_dbg(vdev, MISC, "Fuse: %d tiles enabled. Tile number %d disabled\n",
BTRS_LNL_TILE_MAX_NUM - 1, ffs(config) - 1);
else
ivpu_dbg(vdev, MISC, "Fuse: All %d tiles enabled\n", BTRS_LNL_TILE_MAX_NUM);
*tile_fuse_config = config;
return 0;
}
static int info_init_mtl(struct ivpu_device *vdev)
{
struct ivpu_hw_info *hw = vdev->hw;
hw->tile_fuse = BTRS_MTL_TILE_FUSE_ENABLE_BOTH;
hw->sku = BTRS_MTL_TILE_SKU_BOTH;
hw->config = BTRS_MTL_WP_CONFIG_2_TILE_4_3_RATIO;
hw->sched_mode = ivpu_sched_mode;
return 0;
}
static int info_init_lnl(struct ivpu_device *vdev)
{
struct ivpu_hw_info *hw = vdev->hw;
u32 tile_fuse_config;
int ret;
ret = read_tile_config_fuse(vdev, &tile_fuse_config);
if (ret)
return ret;
hw->sched_mode = ivpu_sched_mode;
hw->tile_fuse = tile_fuse_config;
hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
return 0;
}
int ivpu_hw_btrs_info_init(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return info_init_mtl(vdev);
else
return info_init_lnl(vdev);
}
static int wp_request_sync(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return REGB_POLL_FLD(VPU_HW_BTRS_MTL_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US);
else
return REGB_POLL_FLD(VPU_HW_BTRS_LNL_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US);
}
static int wait_for_status_ready(struct ivpu_device *vdev, bool enable)
{
u32 exp_val = enable ? 0x1 : 0x0;
if (IVPU_WA(punit_disabled))
return 0;
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, READY, exp_val, PLL_TIMEOUT_US);
else
return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, READY, exp_val, PLL_TIMEOUT_US);
}
struct wp_request {
u16 min;
u16 max;
u16 target;
u16 cfg;
u16 epp;
u16 cdyn;
};
static void wp_request_mtl(struct ivpu_device *vdev, struct wp_request *wp)
{
u32 val;
val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, MIN_RATIO, wp->min, val);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, MAX_RATIO, wp->max, val);
REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD0, val);
val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, TARGET_RATIO, wp->target, val);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, EPP, PLL_EPP_DEFAULT, val);
REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD1, val);
val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2, CONFIG, wp->cfg, val);
REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_PAYLOAD2, val);
val = REGB_RD32(VPU_HW_BTRS_MTL_WP_REQ_CMD);
val = REG_SET_FLD(VPU_HW_BTRS_MTL_WP_REQ_CMD, SEND, val);
REGB_WR32(VPU_HW_BTRS_MTL_WP_REQ_CMD, val);
}
static void wp_request_lnl(struct ivpu_device *vdev, struct wp_request *wp)
{
u32 val;
val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, MIN_RATIO, wp->min, val);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, MAX_RATIO, wp->max, val);
REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD0, val);
val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, TARGET_RATIO, wp->target, val);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, EPP, wp->epp, val);
REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD1, val);
val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, CONFIG, wp->cfg, val);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, CDYN, wp->cdyn, val);
REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_PAYLOAD2, val);
val = REGB_RD32(VPU_HW_BTRS_LNL_WP_REQ_CMD);
val = REG_SET_FLD(VPU_HW_BTRS_LNL_WP_REQ_CMD, SEND, val);
REGB_WR32(VPU_HW_BTRS_LNL_WP_REQ_CMD, val);
}
static void wp_request(struct ivpu_device *vdev, struct wp_request *wp)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
wp_request_mtl(vdev, wp);
else
wp_request_lnl(vdev, wp);
}
static int wp_request_send(struct ivpu_device *vdev, struct wp_request *wp)
{
int ret;
ret = wp_request_sync(vdev);
if (ret) {
ivpu_err(vdev, "Failed to sync before workpoint request: %d\n", ret);
return ret;
}
wp_request(vdev, wp);
ret = wp_request_sync(vdev);
if (ret)
ivpu_err(vdev, "Failed to sync after workpoint request: %d\n", ret);
return ret;
}
static void prepare_wp_request(struct ivpu_device *vdev, struct wp_request *wp, bool enable)
{
struct ivpu_hw_info *hw = vdev->hw;
wp->min = hw->pll.min_ratio;
wp->max = hw->pll.max_ratio;
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) {
wp->target = enable ? hw->pll.pn_ratio : 0;
wp->cfg = enable ? hw->config : 0;
wp->cdyn = 0;
wp->epp = 0;
} else {
wp->target = hw->pll.pn_ratio;
wp->cfg = enable ? PLL_CONFIG_DEFAULT : 0;
wp->cdyn = enable ? PLL_CDYN_DEFAULT : 0;
wp->epp = enable ? PLL_EPP_DEFAULT : 0;
}
/* Simics cannot start without at least one tile */
if (enable && ivpu_is_simics(vdev))
wp->cfg = 1;
}
static int wait_for_pll_lock(struct ivpu_device *vdev, bool enable)
{
u32 exp_val = enable ? 0x1 : 0x0;
if (ivpu_hw_btrs_gen(vdev) != IVPU_HW_BTRS_MTL)
return 0;
if (IVPU_WA(punit_disabled))
return 0;
return REGB_POLL_FLD(VPU_HW_BTRS_MTL_PLL_STATUS, LOCK, exp_val, PLL_TIMEOUT_US);
}
int ivpu_hw_btrs_wp_drive(struct ivpu_device *vdev, bool enable)
{
struct wp_request wp;
int ret;
if (IVPU_WA(punit_disabled)) {
ivpu_dbg(vdev, PM, "Skipping workpoint request\n");
return 0;
}
prepare_wp_request(vdev, &wp, enable);
ivpu_dbg(vdev, PM, "PLL workpoint request: %u Hz, config: 0x%x, epp: 0x%x, cdyn: 0x%x\n",
PLL_RATIO_TO_FREQ(wp.target), wp.cfg, wp.epp, wp.cdyn);
ret = wp_request_send(vdev, &wp);
if (ret) {
ivpu_err(vdev, "Failed to send workpoint request: %d\n", ret);
return ret;
}
ret = wait_for_pll_lock(vdev, enable);
if (ret) {
ivpu_err(vdev, "Timed out waiting for PLL lock\n");
return ret;
}
ret = wait_for_status_ready(vdev, enable);
if (ret) {
ivpu_err(vdev, "Timed out waiting for NPU ready status\n");
return ret;
}
return 0;
}
static int d0i3_drive_mtl(struct ivpu_device *vdev, bool enable)
{
int ret;
u32 val;
ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US);
if (ret) {
ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret);
return ret;
}
val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL);
if (enable)
val = REG_SET_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, I3, val);
else
val = REG_CLR_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, I3, val);
REGB_WR32(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, val);
ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US);
if (ret)
ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret);
return ret;
}
static int d0i3_drive_lnl(struct ivpu_device *vdev, bool enable)
{
int ret;
u32 val;
ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US);
if (ret) {
ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret);
return ret;
}
val = REGB_RD32(VPU_HW_BTRS_LNL_D0I3_CONTROL);
if (enable)
val = REG_SET_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, I3, val);
else
val = REG_CLR_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, I3, val);
REGB_WR32(VPU_HW_BTRS_LNL_D0I3_CONTROL, val);
ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US);
if (ret) {
ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret);
return ret;
}
return 0;
}
static int d0i3_drive(struct ivpu_device *vdev, bool enable)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return d0i3_drive_mtl(vdev, enable);
else
return d0i3_drive_lnl(vdev, enable);
}
int ivpu_hw_btrs_d0i3_enable(struct ivpu_device *vdev)
{
int ret;
if (IVPU_WA(punit_disabled))
return 0;
ret = d0i3_drive(vdev, true);
if (ret)
ivpu_err(vdev, "Failed to enable D0i3: %d\n", ret);
udelay(5); /* VPU requires 5 us to complete the transition */
return ret;
}
int ivpu_hw_btrs_d0i3_disable(struct ivpu_device *vdev)
{
int ret;
if (IVPU_WA(punit_disabled))
return 0;
ret = d0i3_drive(vdev, false);
if (ret)
ivpu_err(vdev, "Failed to disable D0i3: %d\n", ret);
return ret;
}
int ivpu_hw_btrs_wait_for_clock_res_own_ack(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return 0;
if (ivpu_is_simics(vdev))
return 0;
return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, CLOCK_RESOURCE_OWN_ACK, 1, TIMEOUT_US);
}
void ivpu_hw_btrs_set_port_arbitration_weights_lnl(struct ivpu_device *vdev)
{
REGB_WR32(VPU_HW_BTRS_LNL_PORT_ARBITRATION_WEIGHTS, WEIGHTS_DEFAULT);
REGB_WR32(VPU_HW_BTRS_LNL_PORT_ARBITRATION_WEIGHTS_ATS, WEIGHTS_ATS_DEFAULT);
}
static int ip_reset_mtl(struct ivpu_device *vdev)
{
int ret;
u32 val;
ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, 0, TIMEOUT_US);
if (ret) {
ivpu_err(vdev, "Timed out waiting for TRIGGER bit\n");
return ret;
}
val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_IP_RESET);
val = REG_SET_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, val);
REGB_WR32(VPU_HW_BTRS_MTL_VPU_IP_RESET, val);
ret = REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_IP_RESET, TRIGGER, 0, TIMEOUT_US);
if (ret)
ivpu_err(vdev, "Timed out waiting for RESET completion\n");
return ret;
}
static int ip_reset_lnl(struct ivpu_device *vdev)
{
int ret;
u32 val;
ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, 0, TIMEOUT_US);
if (ret) {
ivpu_err(vdev, "Wait for *_TRIGGER timed out\n");
return ret;
}
val = REGB_RD32(VPU_HW_BTRS_LNL_IP_RESET);
val = REG_SET_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, val);
REGB_WR32(VPU_HW_BTRS_LNL_IP_RESET, val);
ret = REGB_POLL_FLD(VPU_HW_BTRS_LNL_IP_RESET, TRIGGER, 0, TIMEOUT_US);
if (ret)
ivpu_err(vdev, "Timed out waiting for RESET completion\n");
return ret;
}
int ivpu_hw_btrs_ip_reset(struct ivpu_device *vdev)
{
if (IVPU_WA(punit_disabled))
return 0;
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return ip_reset_mtl(vdev);
else
return ip_reset_lnl(vdev);
}
void ivpu_hw_btrs_profiling_freq_reg_set_lnl(struct ivpu_device *vdev)
{
u32 val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS);
if (vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_DEFAULT)
val = REG_CLR_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, PERF_CLK, val);
else
val = REG_SET_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, PERF_CLK, val);
REGB_WR32(VPU_HW_BTRS_LNL_VPU_STATUS, val);
}
void ivpu_hw_btrs_ats_print_lnl(struct ivpu_device *vdev)
{
ivpu_dbg(vdev, MISC, "Buttress ATS: %s\n",
REGB_RD32(VPU_HW_BTRS_LNL_HM_ATS) ? "Enable" : "Disable");
}
void ivpu_hw_btrs_clock_relinquish_disable_lnl(struct ivpu_device *vdev)
{
u32 val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS);
val = REG_SET_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, DISABLE_CLK_RELINQUISH, val);
REGB_WR32(VPU_HW_BTRS_LNL_VPU_STATUS, val);
}
bool ivpu_hw_btrs_is_idle(struct ivpu_device *vdev)
{
u32 val;
if (IVPU_WA(punit_disabled))
return true;
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) {
val = REGB_RD32(VPU_HW_BTRS_MTL_VPU_STATUS);
return REG_TEST_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, READY, val) &&
REG_TEST_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, IDLE, val);
} else {
val = REGB_RD32(VPU_HW_BTRS_LNL_VPU_STATUS);
return REG_TEST_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, READY, val) &&
REG_TEST_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, IDLE, val);
}
}
int ivpu_hw_btrs_wait_for_idle(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return REGB_POLL_FLD(VPU_HW_BTRS_MTL_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US);
else
return REGB_POLL_FLD(VPU_HW_BTRS_LNL_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US);
}
/* Handler for IRQs from Buttress core (irqB) */
bool ivpu_hw_btrs_irq_handler_mtl(struct ivpu_device *vdev, int irq)
{
u32 status = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_MTL_IRQ_MASK;
bool schedule_recovery = false;
if (!status)
return false;
if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, FREQ_CHANGE, status))
ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq: %08x",
REGB_RD32(VPU_HW_BTRS_MTL_CURRENT_PLL));
if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR, status)) {
ivpu_err(vdev, "ATS_ERR irq 0x%016llx", REGB_RD64(VPU_HW_BTRS_MTL_ATS_ERR_LOG_0));
REGB_WR32(VPU_HW_BTRS_MTL_ATS_ERR_CLEAR, 0x1);
schedule_recovery = true;
}
if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR, status)) {
u32 ufi_log = REGB_RD32(VPU_HW_BTRS_MTL_UFI_ERR_LOG);
ivpu_err(vdev, "UFI_ERR irq (0x%08x) opcode: 0x%02lx axi_id: 0x%02lx cq_id: 0x%03lx",
ufi_log, REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, OPCODE, ufi_log),
REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, AXI_ID, ufi_log),
REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, CQ_ID, ufi_log));
REGB_WR32(VPU_HW_BTRS_MTL_UFI_ERR_CLEAR, 0x1);
schedule_recovery = true;
}
/* This must be done after interrupts are cleared at the source. */
if (IVPU_WA(interrupt_clear_with_0))
/*
* Writing 1 triggers an interrupt, so we can't perform read update write.
* Clear local interrupt status by writing 0 to all bits.
*/
REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, 0x0);
else
REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, status);
if (schedule_recovery)
ivpu_pm_trigger_recovery(vdev, "Buttress IRQ");
return true;
}
/* Handler for IRQs from Buttress core (irqB) */
bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq)
{
u32 status = REGB_RD32(VPU_HW_BTRS_LNL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK;
bool schedule_recovery = false;
if (!status)
return false;
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, status))
ivpu_dbg(vdev, IRQ, "Survivability IRQ\n");
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, FREQ_CHANGE, status))
ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq: %08x", REGB_RD32(VPU_HW_BTRS_LNL_PLL_FREQ));
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, status)) {
ivpu_err(vdev, "ATS_ERR LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n",
REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG1),
REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG2));
REGB_WR32(VPU_HW_BTRS_LNL_ATS_ERR_CLEAR, 0x1);
schedule_recovery = true;
}
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR, status)) {
ivpu_err(vdev, "CFI0_ERR 0x%08x", REGB_RD32(VPU_HW_BTRS_LNL_CFI0_ERR_LOG));
REGB_WR32(VPU_HW_BTRS_LNL_CFI0_ERR_CLEAR, 0x1);
schedule_recovery = true;
}
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR, status)) {
ivpu_err(vdev, "CFI1_ERR 0x%08x", REGB_RD32(VPU_HW_BTRS_LNL_CFI1_ERR_LOG));
REGB_WR32(VPU_HW_BTRS_LNL_CFI1_ERR_CLEAR, 0x1);
schedule_recovery = true;
}
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR, status)) {
ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x",
REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_LOW),
REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_HIGH));
REGB_WR32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_CLEAR, 0x1);
schedule_recovery = true;
}
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR, status)) {
ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x",
REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_LOW),
REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_HIGH));
REGB_WR32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_CLEAR, 0x1);
schedule_recovery = true;
}
/* This must be done after interrupts are cleared at the source. */
REGB_WR32(VPU_HW_BTRS_LNL_INTERRUPT_STAT, status);
if (schedule_recovery)
ivpu_pm_trigger_recovery(vdev, "Buttress IRQ");
return true;
}
static void dct_drive_40xx(struct ivpu_device *vdev, u32 dct_val)
{
u32 val = REGB_RD32(VPU_HW_BTRS_LNL_PCODE_MAILBOX);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX, CMD, DCT_REQ, val);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX, PARAM1,
dct_val ? DCT_ENABLE : DCT_DISABLE, val);
val = REG_SET_FLD_NUM(VPU_HW_BTRS_LNL_PCODE_MAILBOX, PARAM2, dct_val, val);
REGB_WR32(VPU_HW_BTRS_LNL_PCODE_MAILBOX, val);
}
void ivpu_hw_btrs_dct_drive(struct ivpu_device *vdev, u32 dct_val)
{
return dct_drive_40xx(vdev, dct_val);
}
static u32 pll_ratio_to_freq_mtl(u32 ratio, u32 config)
{
u32 pll_clock = PLL_REF_CLK_FREQ * ratio;
u32 cpu_clock;
if ((config & 0xff) == MTL_PLL_RATIO_4_3)
cpu_clock = pll_clock * 2 / 4;
else
cpu_clock = pll_clock * 2 / 5;
return cpu_clock;
}
u32 ivpu_hw_btrs_ratio_to_freq(struct ivpu_device *vdev, u32 ratio)
{
struct ivpu_hw_info *hw = vdev->hw;
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return pll_ratio_to_freq_mtl(ratio, hw->config);
else
return PLL_RATIO_TO_FREQ(ratio);
}
static u32 pll_freq_get_mtl(struct ivpu_device *vdev)
{
u32 pll_curr_ratio;
pll_curr_ratio = REGB_RD32(VPU_HW_BTRS_MTL_CURRENT_PLL);
pll_curr_ratio &= VPU_HW_BTRS_MTL_CURRENT_PLL_RATIO_MASK;
if (!ivpu_is_silicon(vdev))
return PLL_SIMULATION_FREQ;
return pll_ratio_to_freq_mtl(pll_curr_ratio, vdev->hw->config);
}
static u32 pll_freq_get_lnl(struct ivpu_device *vdev)
{
u32 pll_curr_ratio;
pll_curr_ratio = REGB_RD32(VPU_HW_BTRS_LNL_PLL_FREQ);
pll_curr_ratio &= VPU_HW_BTRS_LNL_PLL_FREQ_RATIO_MASK;
return PLL_RATIO_TO_FREQ(pll_curr_ratio);
}
u32 ivpu_hw_btrs_pll_freq_get(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return pll_freq_get_mtl(vdev);
else
return pll_freq_get_lnl(vdev);
}
u32 ivpu_hw_btrs_telemetry_offset_get(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_OFFSET);
else
return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_OFFSET);
}
u32 ivpu_hw_btrs_telemetry_size_get(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_SIZE);
else
return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_SIZE);
}
u32 ivpu_hw_btrs_telemetry_enable_get(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return REGB_RD32(VPU_HW_BTRS_MTL_VPU_TELEMETRY_ENABLE);
else
return REGB_RD32(VPU_HW_BTRS_LNL_VPU_TELEMETRY_ENABLE);
}
void ivpu_hw_btrs_global_int_disable(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x1);
else
REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x1);
}
void ivpu_hw_btrs_global_int_enable(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x0);
else
REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x0);
}
void ivpu_hw_btrs_irq_enable(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) {
REGB_WR32(VPU_HW_BTRS_MTL_LOCAL_INT_MASK, (u32)(~BTRS_MTL_IRQ_MASK));
REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x0);
} else {
REGB_WR32(VPU_HW_BTRS_LNL_LOCAL_INT_MASK, (u32)(~BTRS_LNL_IRQ_MASK));
REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x0);
}
}
void ivpu_hw_btrs_irq_disable(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) {
REGB_WR32(VPU_HW_BTRS_MTL_GLOBAL_INT_MASK, 0x1);
REGB_WR32(VPU_HW_BTRS_MTL_LOCAL_INT_MASK, BTRS_IRQ_DISABLE_MASK);
} else {
REGB_WR32(VPU_HW_BTRS_LNL_GLOBAL_INT_MASK, 0x1);
REGB_WR32(VPU_HW_BTRS_LNL_LOCAL_INT_MASK, BTRS_IRQ_DISABLE_MASK);
}
}
static void diagnose_failure_mtl(struct ivpu_device *vdev)
{
u32 reg = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_MTL_IRQ_MASK;
if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR, reg))
ivpu_err(vdev, "ATS_ERR irq 0x%016llx", REGB_RD64(VPU_HW_BTRS_MTL_ATS_ERR_LOG_0));
if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, UFI_ERR, reg)) {
u32 log = REGB_RD32(VPU_HW_BTRS_MTL_UFI_ERR_LOG);
ivpu_err(vdev, "UFI_ERR irq (0x%08x) opcode: 0x%02lx axi_id: 0x%02lx cq_id: 0x%03lx",
log, REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, OPCODE, log),
REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, AXI_ID, log),
REG_GET_FLD(VPU_HW_BTRS_MTL_UFI_ERR_LOG, CQ_ID, log));
}
}
static void diagnose_failure_lnl(struct ivpu_device *vdev)
{
u32 reg = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK;
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, reg)) {
ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n",
REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG1),
REGB_RD32(VPU_HW_BTRS_LNL_ATS_ERR_LOG2));
}
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI0_ERR, reg))
ivpu_err(vdev, "CFI0_ERR_LOG 0x%08x\n", REGB_RD32(VPU_HW_BTRS_LNL_CFI0_ERR_LOG));
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, CFI1_ERR, reg))
ivpu_err(vdev, "CFI1_ERR_LOG 0x%08x\n", REGB_RD32(VPU_HW_BTRS_LNL_CFI1_ERR_LOG));
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR0_ERR, reg))
ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x\n",
REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_LOW),
REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI0_HIGH));
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, IMR1_ERR, reg))
ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x\n",
REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_LOW),
REGB_RD32(VPU_HW_BTRS_LNL_IMR_ERR_CFI1_HIGH));
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, SURV_ERR, reg))
ivpu_err(vdev, "Survivability IRQ\n");
}
void ivpu_hw_btrs_diagnose_failure(struct ivpu_device *vdev)
{
if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
return diagnose_failure_mtl(vdev);
else
return diagnose_failure_lnl(vdev);
}

46
drivers/accel/ivpu/ivpu_hw_btrs.h Normal file
View File

@ -0,0 +1,46 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2020-2024 Intel Corporation
*/
#ifndef __IVPU_HW_BTRS_H__
#define __IVPU_HW_BTRS_H__
#include "ivpu_drv.h"
#include "ivpu_hw_37xx_reg.h"
#include "ivpu_hw_40xx_reg.h"
#include "ivpu_hw_reg_io.h"
#define PLL_PROFILING_FREQ_DEFAULT 38400000
#define PLL_PROFILING_FREQ_HIGH 400000000
#define PLL_RATIO_TO_FREQ(x) ((x) * PLL_REF_CLK_FREQ)
int ivpu_hw_btrs_info_init(struct ivpu_device *vdev);
void ivpu_hw_btrs_freq_ratios_init(struct ivpu_device *vdev);
int ivpu_hw_btrs_irqs_clear_with_0_mtl(struct ivpu_device *vdev);
int ivpu_hw_btrs_wp_drive(struct ivpu_device *vdev, bool enable);
int ivpu_hw_btrs_wait_for_clock_res_own_ack(struct ivpu_device *vdev);
int ivpu_hw_btrs_d0i3_enable(struct ivpu_device *vdev);
int ivpu_hw_btrs_d0i3_disable(struct ivpu_device *vdev);
void ivpu_hw_btrs_set_port_arbitration_weights_lnl(struct ivpu_device *vdev);
bool ivpu_hw_btrs_is_idle(struct ivpu_device *vdev);
int ivpu_hw_btrs_wait_for_idle(struct ivpu_device *vdev);
int ivpu_hw_btrs_ip_reset(struct ivpu_device *vdev);
void ivpu_hw_btrs_profiling_freq_reg_set_lnl(struct ivpu_device *vdev);
void ivpu_hw_btrs_ats_print_lnl(struct ivpu_device *vdev);
void ivpu_hw_btrs_clock_relinquish_disable_lnl(struct ivpu_device *vdev);
bool ivpu_hw_btrs_irq_handler_mtl(struct ivpu_device *vdev, int irq);
bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq);
void ivpu_hw_btrs_dct_drive(struct ivpu_device *vdev, u32 dct_val);
u32 ivpu_hw_btrs_pll_freq_get(struct ivpu_device *vdev);
u32 ivpu_hw_btrs_ratio_to_freq(struct ivpu_device *vdev, u32 ratio);
u32 ivpu_hw_btrs_telemetry_offset_get(struct ivpu_device *vdev);
u32 ivpu_hw_btrs_telemetry_size_get(struct ivpu_device *vdev);
u32 ivpu_hw_btrs_telemetry_enable_get(struct ivpu_device *vdev);
void ivpu_hw_btrs_global_int_enable(struct ivpu_device *vdev);
void ivpu_hw_btrs_global_int_disable(struct ivpu_device *vdev);
void ivpu_hw_btrs_irq_enable(struct ivpu_device *vdev);
void ivpu_hw_btrs_irq_disable(struct ivpu_device *vdev);
void ivpu_hw_btrs_diagnose_failure(struct ivpu_device *vdev);
#endif /* __IVPU_HW_BTRS_H__ */

1174
drivers/accel/ivpu/ivpu_hw_ip.c Normal file
View File

File diff suppressed because it is too large Load Diff

36
drivers/accel/ivpu/ivpu_hw_ip.h Normal file
View File

@ -0,0 +1,36 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2020-2024 Intel Corporation
*/
#ifndef __IVPU_HW_IP_H__
#define __IVPU_HW_IP_H__
#include "ivpu_drv.h"
int ivpu_hw_ip_host_ss_configure(struct ivpu_device *vdev);
void ivpu_hw_ip_idle_gen_enable(struct ivpu_device *vdev);
void ivpu_hw_ip_idle_gen_disable(struct ivpu_device *vdev);
int ivpu_hw_ip_pwr_domain_enable(struct ivpu_device *vdev);
int ivpu_hw_ip_host_ss_axi_enable(struct ivpu_device *vdev);
int ivpu_hw_ip_top_noc_enable(struct ivpu_device *vdev);
u64 ivpu_hw_ip_read_perf_timer_counter(struct ivpu_device *vdev);
void ivpu_hw_ip_snoop_disable(struct ivpu_device *vdev);
void ivpu_hw_ip_tbu_mmu_enable(struct ivpu_device *vdev);
int ivpu_hw_ip_soc_cpu_boot(struct ivpu_device *vdev);
void ivpu_hw_ip_wdt_disable(struct ivpu_device *vdev);
void ivpu_hw_ip_diagnose_failure(struct ivpu_device *vdev);
u32 ivpu_hw_ip_ipc_rx_count_get(struct ivpu_device *vdev);
void ivpu_hw_ip_irq_clear(struct ivpu_device *vdev);
bool ivpu_hw_ip_irq_handler_37xx(struct ivpu_device *vdev, int irq, bool *wake_thread);
bool ivpu_hw_ip_irq_handler_40xx(struct ivpu_device *vdev, int irq, bool *wake_thread);
void ivpu_hw_ip_db_set(struct ivpu_device *vdev, u32 db_id);
u32 ivpu_hw_ip_ipc_rx_addr_get(struct ivpu_device *vdev);
void ivpu_hw_ip_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr);
void ivpu_hw_ip_irq_enable(struct ivpu_device *vdev);
void ivpu_hw_ip_irq_disable(struct ivpu_device *vdev);
void ivpu_hw_ip_diagnose_failure(struct ivpu_device *vdev);
void ivpu_hw_ip_fabric_req_override_enable_50xx(struct ivpu_device *vdev);
void ivpu_hw_ip_fabric_req_override_disable_50xx(struct ivpu_device *vdev);
#endif /* __IVPU_HW_IP_H__ */

6
drivers/accel/ivpu/ivpu_ipc.c
View File

@ -129,7 +129,7 @@ static void ivpu_ipc_tx_release(struct ivpu_device *vdev, u32 vpu_addr)
static void ivpu_ipc_tx(struct ivpu_device *vdev, u32 vpu_addr) static void ivpu_ipc_tx(struct ivpu_device *vdev, u32 vpu_addr)
{ {
ivpu_hw_reg_ipc_tx_set(vdev, vpu_addr); ivpu_hw_ipc_tx_set(vdev, vpu_addr);
} }
static void static void
@ -392,8 +392,8 @@ void ivpu_ipc_irq_handler(struct ivpu_device *vdev, bool *wake_thread)
* Driver needs to purge all messages from IPC FIFO to clear IPC interrupt. * Driver needs to purge all messages from IPC FIFO to clear IPC interrupt.
* Without purge IPC FIFO to 0 next IPC interrupts won't be generated. * Without purge IPC FIFO to 0 next IPC interrupts won't be generated.
*/ */
while (ivpu_hw_reg_ipc_rx_count_get(vdev)) { while (ivpu_hw_ipc_rx_count_get(vdev)) {
vpu_addr = ivpu_hw_reg_ipc_rx_addr_get(vdev); vpu_addr = ivpu_hw_ipc_rx_addr_get(vdev);
if (vpu_addr == REG_IO_ERROR) { if (vpu_addr == REG_IO_ERROR) {
ivpu_err_ratelimited(vdev, "Failed to read IPC rx addr register\n"); ivpu_err_ratelimited(vdev, "Failed to read IPC rx addr register\n");
return; return;

2
drivers/accel/ivpu/ivpu_job.c
View File

@ -27,7 +27,7 @@
static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq) static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)
{ {
ivpu_hw_reg_db_set(vdev, cmdq->db_id); ivpu_hw_db_set(vdev, cmdq->db_id);
} }
static int ivpu_preemption_buffers_create(struct ivpu_device *vdev, static int ivpu_preemption_buffers_create(struct ivpu_device *vdev,