ASP.NET Core Web Api - Problem with using session

Question

I am developing an asp.net core web api and I want to store a token that will be sent to my endpoints in order for a user to authenticate. For that I have some requirements which force me to implement an own authentication method. I inherit from AuthenticationHandler and implement the HandleAuthenticateAsync method:

public AuthenticateHandler(
    IOptionsMonitor<AuthenticationSchemeOptions> options, 
    ILoggerFactory logger, 
    UrlEncoder encoder, 
    ISystemClock clock,
    IHttpContextAccessor httpContextAccessor) 
    : base(options, logger, encoder, clock)
{
    AuthenticateHandlerHelperFunctions = new AuthenticateHandlerHelperFunctions();
    _checkAccessTokenModel = new CheckAccessTokenModel();
    session = httpContextAccessor.HttpContext.Session;
}

protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
{

    //before this: check header and get authorization informations
    string submittedToken = authorizationheader.Substring("bearer".Length).Trim();
    try
    {
        if (string.IsNullOrEmpty(session.GetString("Token")))
        {
            _checkAccessTokenModel = await AuthenticateHandlerHelperFunctions.CheckAccessToken(submittedToken);
            if(_checkAccessTokenModel.Active == false)
            {
                _failReason = "Token not valid anymore, request another one!";
                return AuthenticateResult.Fail("Token not valid anymore, request another one!");
            }
            session.SetString("Token", submittedToken);
        }
    }
    catch
    {
        return AuthenticateResult.Fail("Invalid Authorization Header");
    }

    var claims = new[] {
        new Claim(ClaimTypes.Name, _checkAccessTokenModel.Exp.ToString()),
    };
    var identity = new ClaimsIdentity(claims, Scheme.Name);
    var principal = new ClaimsPrincipal(identity);
    var ticket = new AuthenticationTicket(principal, Scheme.Name);

    return AuthenticateResult.Success(ticket);
}

The goal is to use the session to save the token and not execute the CheckAccessToken method for every request. I will get frequent data on the endpoints that are configured with [Authorize] so I want to save computing time. I looked this up and most of the errors where problems with the startup where the app.UseSession() was not set correctly etc. but not in my case I believe. Here is my Startup:

 public void ConfigureServices(IServiceCollection services)
{
    services.AddSwaggerGen(c =>
    {
        c.SwaggerDoc("v1", new OpenApiInfo { Title = "DigVPapi", Version = "v1" });
    });

    services.AddDbContextFactory<AntragDBNoInheritanceContext>(options =>
    {
        options.UseSqlServer(Configuration.GetConnectionString("DefaultConnection"));
    });

    services.AddAuthentication("BasicAuthentication")
        .AddScheme<AuthenticationSchemeOptions, AuthenticateHandler>("BasicAuthentication", null);

    services.AddDistributedMemoryCache();
    services.AddSession(options =>
    {
        options.IdleTimeout = System.TimeSpan.FromSeconds(10);
        options.Cookie.HttpOnly = true;
        options.Cookie.IsEssential = true;
    });
    services.AddHttpContextAccessor();

    services.AddSingleton<IJWTManagerRepository, JWTManagerRepository>();
    services.AddControllers();
}

// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
    if (env.IsDevelopment())
    {
        app.UseDeveloperExceptionPage();
        app.UseSwagger();
        app.UseSwaggerUI(c => c.SwaggerEndpoint("/swagger/v1/swagger.json", "DigVPapi v1"));
    }

    app.UseHttpsRedirection();

    app.UseRouting();
            
    app.UseAuthentication();
    app.UseAuthorization();
    app.UseSession();

    app.UseEndpoints(endpoints =>
    {
        endpoints.MapControllers();
    });
}

If this is not possible. What could I do instead to save the token in some different way? Of course I could save the token in the database but this would only move the problem to a database query tha twould be made every time. The error that I get when trying to authenticate is following

System.InvalidOperationException: 'Session has not been configured for this application or request.'