How To increase the records to be shown in ElasticaSearch in c#

Question

it's Defaulty Showing 10000 and i want to show more than that

this is what is tried

this the method for this

public JsonResult GetUsageLog()
        {
            List<AcctountSession> data = new List<AcctountSession>();
            ISearchResponse<AcctountSession> responsedata = null;
            var query = new object();
    
            try
            {
                var defaultIndex = "all-radius-session-*";

                var settings = new ConnectionSettings(new Uri(allElasticUrl)).DefaultIndex("acctountsessionid");
                ElasticClient Elastic = new ElasticClient(settings);


                var sqlResponse = Elastic.Sql.Query(q => q.Query("SELECT count(0) from all-radius-session-* WHERE timestamp>='2022-03-01T00:00:00' and timestamp='2022-04-01T00:00:00'"));

            
                if (Elastic.Indices.Exists(defaultIndex).Exists)
                {
                    responsedata = Elastic.Search<AcctountSession>(s => s
                               .Index(defaultIndex)
                               .Size(10000)
                               .TrackTotalHits(true)
                               .Query(q => q.DateRange(r => r
                               .Field(p => p.Timestamp)))
                               .Sort(p => p.Descending("@timestamp")));
                    data = (from hits in responsedata.Hits select hits.Source).ToList();

                    query = data.GroupBy(x => new { x.Timestamp.Date, x.cust_id, x.snl_dev_id, x.ftc_code, x.service_type })
                                .Select(group => new
                                {
                                    date = Convert.ToDateTime(group.Key.Date).ToString("dd-MM-yyyy"),
                                    cust_id = group.Key.cust_id,
                                    snl_dev_id = group.Key.snl_dev_id,
                                    ftc_code= group.Key.ftc_code,
                                    service_type = group.Key.service_type,
                                    acctountinputoctets = group.Sum(x => x.acctountinputoctets ),
                                    acctountoutputoctets = group.Sum(x => x.acctountoutputoctets ),
                                    acctipv6inputoctets = group.Sum(x => ((x.acctipv6inputoctets != null && x.acctipv6inputoctets != "") ? Int64.Parse(x.acctipv6inputoctets) : Int64.Parse("0"))),
                                    acctipv6outputoctets = group.Sum(x => ((x.acctipv6outputoctets != null && x.acctipv6inputoctets != "") ? Int64.Parse(x.acctipv6outputoctets) : Int64.Parse("0")))
                                });

                }
            }
            catch (Exception ex)
            {

            }
            var jsonResult = Json(query, JsonRequestBehavior.AllowGet);
            jsonResult.MaxJsonLength = int.MaxValue;
            return jsonResult;
        }

How could I go about getting all the logs if lets say I have more than 10K logs/results with the latest version of Elasticsearch

In this how can i try to modify it to give more than 10000

What Can i try for this

Answer 1

You can use Scrolling API of Elasticsearch to retrive more then 10k documents from index.

Below is simple example:

var searchResponse = Client.Search<Project>(s => s
    .Query(q => q
        .Term(f => f.State, StateOfBeing.Stable)
    )
    .Scroll("10s") 
);

while (searchResponse.Documents.Any()) 
{
    ProcessResponse(searchResponse); 
    searchResponse = Client.Scroll<Project>("10s", searchResponse.ScrollId);
}