'postgresql evaluate large table with sums and group bys

I have a large table (8Mio rs) in my postgres DB and I want to evaluate multiple columns (over 30 vals with filters) grouped by some other columns (grps) with the same technique. I want to see the given result for each possible combination, that means also combinations that doesnt return anything. It is neccessary for a better understanding of the data. I tried several approaches for implementation, for example the following one (and one with multiple LOOPs and an insert statement). It works both ways but the performance isn't too good and the query gets quite difficoult to read. If you could give me some advice for improvement, I would be really happy!

DROP TABLE IF EXISTS testdata;
CREATE TABLE testdata (grp1 text, grp2 text, val1 int, val2 int, val3 int, val4 int, val5 int);

INSERT INTO testdata VALUES ('A', 'X', 1,2,3,4,5);
INSERT INTO testdata VALUES ('A', 'X', 3,3,3,4,5);
INSERT INTO testdata VALUES ('A', 'X', 4,2,3,4,5);
INSERT INTO testdata VALUES ('A', 'Y', 4,2,3,4,5);
INSERT INTO testdata VALUES ('A', 'Y', 3,2,3,4,5);
INSERT INTO testdata VALUES ('B', 'X', 5,2,3,4,5);
--SELECT * FROM testdata;

SELECT g1.*, g2.*, val.*, v.small, v.medium, v.large
FROM (
    SELECT DISTINCT grp1 FROM testdata) g1
CROSS JOIN (
    SELECT DISTINCT grp2 FROM testdata) g2
CROSS JOIN (
    SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'testdata' AND column_name LIKE 'val%') val

LEFT JOIN (
    SELECT grp1, grp2, 'val1'::text AS val,
        COUNT(*) FILTER (WHERE val1 < 3) as small, 
        COUNT(*) FILTER (WHERE val1 = 3) as medium, 
        COUNT(*) FILTER (WHERE val1 > 3) AS large
    FROM testdata GROUP By grp1, grp2
    UNION
    SELECT grp1, grp2, 'val2'::text AS val,
        COUNT(*) FILTER (WHERE val2 < 3) as small, 
        COUNT(*) FILTER (WHERE val2 = 3) as medium, 
        COUNT(*) FILTER (WHERE val2 > 3) AS large
    FROM testdata GROUP By grp1, grp2
    UNION
    SELECT grp1, grp2, 'val3'::text AS val,
        COUNT(*) FILTER (WHERE val3 < 3) as small, 
        COUNT(*) FILTER (WHERE val3 = 3) as medium, 
        COUNT(*) FILTER (WHERE val3 > 3) AS large
    FROM testdata GROUP By grp1, grp2
    UNION
    SELECT grp1, grp2, 'val4'::text AS val,
        COUNT(*) FILTER (WHERE val4 < 3) as small, 
        COUNT(*) FILTER (WHERE val4 = 3) as medium, 
        COUNT(*) FILTER (WHERE val4 > 3) AS large
    FROM testdata GROUP By grp1, grp2
    UNION
    SELECT grp1, grp2, 'val5'::text AS val,
        COUNT(*) FILTER (WHERE val5 < 3) as small, 
        COUNT(*) FILTER (WHERE val5 = 3) as medium, 
        COUNT(*) FILTER (WHERE val5 > 3) AS large
    FROM testdata GROUP By grp1, grp2
    -- more values
) v
  ON g1.grp1 = v.grp1 AND g2.grp2 = v.grp2 AND val.column_name = v.val
ORDER BY g1.grp1, g2.grp2;


Sources

This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.

Source: Stack Overflow

Solution Source