Re: [kvm-unit-tests PATCH v3 08/14] arm/arm64: ITS: its_enable_defaults

[Auger Eric]

Hi Drew,

On 2/7/20 1:41 PM, Andrew Jones wrote:
> On Tue, Jan 28, 2020 at 11:34:53AM +0100, Eric Auger wrote:
>> its_enable_defaults() is the top init function that allocates the
>> command queue and all the requested tables (device, collection,
>> lpi config and pending tables), enable LPIs at distributor level
>> and ITS level.
>>
>> gicv3_enable_defaults must be called before.
>>
>> Signed-off-by: Eric Auger <address@hidden>
>>
>> ---
>>
>> v2 -> v3:
>> - introduce its_setup_baser in this patch
>> - squash "arm/arm64: ITS: Init the command queue" in this patch.
>> ---
>>  lib/arm/asm/gic-v3-its.h |  8 ++++
>>  lib/arm/gic-v3-its.c     | 89 ++++++++++++++++++++++++++++++++++++++++
>>  2 files changed, 97 insertions(+)
>>
>> diff --git a/lib/arm/asm/gic-v3-its.h b/lib/arm/asm/gic-v3-its.h
>> index 815c515..fe73c04 100644
>> --- a/lib/arm/asm/gic-v3-its.h
>> +++ b/lib/arm/asm/gic-v3-its.h
>> @@ -36,6 +36,8 @@ struct its_data {
>>      void *base;
>>      struct its_typer typer;
>>      struct its_baser baser[GITS_BASER_NR_REGS];
>> +    struct its_cmd_block *cmd_base;
>> +    struct its_cmd_block *cmd_write;
>>  };
>>  
>>  extern struct its_data its_data;
>> @@ -88,10 +90,16 @@ extern struct its_data its_data;
>>  #define GITS_BASER_TYPE_DEVICE              1
>>  #define GITS_BASER_TYPE_COLLECTION  4
>>  
>> +
>> +struct its_cmd_block {
>> +    u64 raw_cmd[4];
>> +};
>> +
>>  extern void its_parse_typer(void);
>>  extern void its_init(void);
>>  extern int its_parse_baser(int i, struct its_baser *baser);
>>  extern struct its_baser *its_lookup_baser(int type);
>> +extern void its_enable_defaults(void);
>>  
>>  #else /* __arm__ */
>>  
>> diff --git a/lib/arm/gic-v3-its.c b/lib/arm/gic-v3-its.c
>> index 2c0ce13..d1e7e52 100644
>> --- a/lib/arm/gic-v3-its.c
>> +++ b/lib/arm/gic-v3-its.c
>> @@ -86,3 +86,92 @@ void its_init(void)
>>              its_parse_baser(i, &its_data.baser[i]);
>>  }
>>  
>> +static void its_setup_baser(int i, struct its_baser *baser)
>> +{
>> +    unsigned long n = (baser->nr_pages * baser->psz) >> PAGE_SHIFT;
>> +    unsigned long order = is_power_of_2(n) ? fls(n) : fls(n) + 1;
>> +    u64 val;
>> +
>> +    baser->table_addr = (u64)virt_to_phys(alloc_pages(order));
>> +
>> +    val = ((u64)baser->table_addr                                   |
>> +            ((u64)baser->type       << GITS_BASER_TYPE_SHIFT)       |
>> +            ((u64)(baser->esz - 1)  << GITS_BASER_ENTRY_SIZE_SHIFT) |
>> +            ((baser->nr_pages - 1)  << GITS_BASER_PAGES_SHIFT)      |
>> +            (u64)baser->indirect    << 62                           |
>> +            (u64)baser->valid       << 63);
> 
>  << GITS_BASER_INDIRECT
>  << GITS_BASER_VALID
Those are BIT()
        if (baser->indirect)
                val |= GITS_BASER_INDIRECT;
        if (baser->valid)
                val |= GITS_BASER_VALID;

> 
>> +
>> +    switch (baser->psz) {
>> +    case SZ_4K:
>> +            val |= GITS_BASER_PAGE_SIZE_4K;
>> +            break;
>> +    case SZ_16K:
>> +            val |= GITS_BASER_PAGE_SIZE_16K;
>> +            break;
>> +    case SZ_64K:
>> +            val |= GITS_BASER_PAGE_SIZE_64K;
>> +            break;
>> +    }
>> +
>> +    writeq(val, gicv3_its_base() + GITS_BASER + i * 8);
>> +}
>> +
>> +/**
>> + * init_cmd_queue: Allocate the command queue and initialize
>> + * CBASER, CREADR, CWRITER
>> + */
>> +static void its_cmd_queue_init(void)
>> +{
>> +    unsigned long n = SZ_64K >> PAGE_SHIFT;
>> +    unsigned long order = fls(n);
>> +    u64 cbaser;
>> +
>> +    its_data.cmd_base = (void *)virt_to_phys(alloc_pages(order));
>> +
>> +    cbaser = ((u64)its_data.cmd_base | (SZ_64K / SZ_4K - 1) | 
>> GITS_CBASER_VALID);
>> +
>> +    writeq(cbaser, its_data.base + GITS_CBASER);
>> +
>> +    its_data.cmd_write = its_data.cmd_base;
>> +    writeq(0, its_data.base + GITS_CWRITER);
>> +}
>> +
>> +void its_enable_defaults(void)
>> +{
>> +    unsigned int i;
>> +
>> +    its_parse_typer();
>> +
>> +    /* Allocate BASER tables (device and collection tables) */
>> +    for (i = 0; i < GITS_BASER_NR_REGS; i++) {
>> +            struct its_baser *baser = &its_data.baser[i];
>> +            int ret;
>> +
>> +            ret = its_parse_baser(i, baser);
>> +            if (ret)
>> +                    continue;
> 
> Didn't we already parse typer and baser at its_init time? How/why would
> its_parse_baser fail? Should we assert when it does?
Yes we parsed them already so I can directly use
baser = &its_data.baser[i];

More generally I have simplified and only cares about the device        and
collection baser now.
> 
>> +
>> +            switch (baser->type) {
>> +            case GITS_BASER_TYPE_DEVICE:
>> +                    baser->valid = true;
>> +                    its_setup_baser(i, baser);
>> +                    break;
>> +            case GITS_BASER_TYPE_COLLECTION:
>> +                    baser->valid = true;
>> +                    its_setup_baser(i, baser);
>> +                    break;
>> +            default:
>> +                    break;
> 
> assert() ?
> 
>> +            }
>> +    }
>> +
>> +    /* Allocate LPI config and pending tables */
>> +    gicv3_lpi_alloc_tables();
>> +
>> +    its_cmd_queue_init();
>> +
>> +    for (i = 0; i < nr_cpus; i++)
>> +            gicv3_lpi_rdist_ctrl(i, true);
>> +
>> +    writel(GITS_CTLR_ENABLE, its_data.base + GITS_CTLR);
>> +}
>> -- 
>> 2.20.1
>>
>>
> 
> It's looking like we don't have a clean separation between its_init and
> its_enable_defaults. I'd expect its_init to do all the allocating of
> memory and its_enable_defaults to do all the write's to the device.
> We should only do its_init once and its_enable should be something
> we can do again (after a disable, on reset, etc.). Is that not possible
> with the ITS device?

I can moved the BASER device & collection table allocation, + the queue
init in its_init()

However gicv3_lpi_alloc_tables() must be called after
gicv3_enable_defaults as it uses the redist_base[] initialized there.

Thanks

Eric


> 
> Thanks,
> drew
>