Get error "Cannot redirect to the authorization endpoint, the configuration may be missing or invalid" while using authority url in password flow

Question

I want to use password flow in order to authenticate my MVC resource server by a separate auth server.

Currently, Token can be made directly by calling token endpoint, but when I open a secured MVC action, It throws me

Cannot redirect to the authorization endpoint, the configuration may be missing or invalid

It is clear that auth server is looking at the Authorization endpoint, but what I wondering is, how to implement it without that endpoint.

My auth server configuration is :

AddServer(options =>
        {
            // Enable the token endpoint.
            options.SetTokenEndpointUris("/token");
            // Enable the password flow.
            options.AllowPasswordFlow()
                .AllowRefreshTokenFlow();


            //// Encryption and signing of tokens

            options.AddDevelopmentEncryptionCertificate()
                .AddDevelopmentSigningCertificate();
            // Register the ASP.NET Core host and configure the ASP.NET Core options.
            options.UseAspNetCore()
                .EnableTokenEndpointPassthrough()
                .DisableTransportSecurityRequirement();


            //Disable encryption to Debug purposes 
            if (env.IsDevelopment())
                options.DisableAccessTokenEncryption();


            options.SetRefreshTokenLifetime(
                TimeSpan.FromDays(int.Parse(config.GetSection("OpenId:RefreshTokenExpireFromDay").Value)));
            options.SetAccessTokenLifetime(
                TimeSpan.FromHours(int.Parse(config.GetSection("OpenId:AccessTokenExpireFromHour").Value)));

            options.DisableRollingRefreshTokens();
            options.UseReferenceRefreshTokens();

           // options.SetIssuer(new Uri("http://localhost:8001"));

        })
        // Register the OpenIddict validation components.
        .AddValidation(options =>
        {
            // Import the configuration from the local OpenIddict server instance.
            options.UseLocalServer();

            // Register the ASP.NET Core host.
            options.UseAspNetCore();

        });
    services.Configure<IdentityOptions>(options =>
    {
        options.ClaimsIdentity.UserNameClaimType = OpenIddictConstants.Claims.Username;
        options.ClaimsIdentity.UserIdClaimType = OpenIddictConstants.Claims.Subject;
        options.ClaimsIdentity.RoleClaimType = OpenIddictConstants.Claims.Role;
        options.ClaimsIdentity.EmailClaimType = OpenIddictConstants.Claims.Email;
        options.ClaimsIdentity.SecurityStampClaimType = "secret_value";
    });

And resource server configuraion is :

  .AddOpenIdConnect(options =>
    {
        options.SignInScheme = "Cookies";
        options.Authority = (env.IsProduction() ? "https://" : "http://") + openIDConnectSettings["Authority"];
        options.ClientId = openIDConnectSettings["ClientId"];
        options.ClientSecret = openIDConnectSettings["ClientSecret"];
        options.ResponseType = OpenIdConnectResponseType.Token;
        options.UsePkce = true;
        options.Scope.Add("profile");
        options.SaveTokens = true;
        options.GetClaimsFromUserInfoEndpoint = true;
        options.RequireHttpsMetadata = false;
    });