Flutter widget not rebuild when state is updated

Question

I am trying to build an app where you can draw with a stylus on images.

I am using the Listener widget to listen for stylus inputs to save the offsets in a _points variable to draw those the canvas with a CustomPainter. I can also erase the strokes I made in eraserMode = true with the stylus by removing points from the _points list. On top of the Listener I have a InteractiveViewer widget to be able to zoom in on the image. But here comes the problem. Whenever I set panEnabled in the ÌnteractiveViewer to false, so that the stylus does not move the image, the widget does not get rebuild when I remove offsets from the _points list. It only gets rebuild after I switch the eraserMode to false. The drawing still works as intended. If panEnabled is set to true, removing the points works instantly but then the stylus also moves the image.

The code looks like this

import 'package:flutter/material.dart';
import 'dart:ui' as ui;
import 'utils.dart';
import 'workSheetDrawer.dart';
import 'pdf_image_info.dart';

class DigitalInkView extends StatefulWidget {
  const DigitalInkView({Key? key, required this.image, required this.pages})
      : super(key: key);

  final ui.Image image;
  final int pages;

  @override
  _DigitalInkViewState createState() => _DigitalInkViewState();
}

class _DigitalInkViewState extends State<DigitalInkView> {
  List<Offset?> _points = <Offset>[];
  Offset _offset = const Offset(0, 0);
  bool eraserMode = false;

  @override
  void initState() {
    super.initState();
  }

  @override
  Widget build(BuildContext context) {
    // Init PDFImageInfo object
    PDFImageInfo imageInfo = PDFImageInfo(
        image: widget.image, context: context, pages: widget.pages);

    Listener listener = Listener(
        onPointerMove: _updatePoints,
        onPointerUp: _addNull,
        child: CustomPaint(
            size: Size.infinite,
            painter: WorkSheetDrawer(
                points: _points,
                markerPoints: _markerPoints,
                imageInfo: imageInfo,
                offset: _offset,
                markerMode: markerMode)));

    return Scaffold(
      appBar: AppBar(
        title: const Text('Handschrift Erkennung'),
        toolbarHeight: 56,
      ),
      body: SafeArea(
        child: Column(
          children: [
            Expanded(
                child: InteractiveViewer(
                    boundaryMargin: const EdgeInsets.all(0.0),
                    minScale: 1.01,
                    maxScale: 3.0,
                    panEnabled: false,  //When this is set to true erasing works.
                                        //When set to false, the widget only rebuilds
                                        //after setting eraserMode to false
                    scaleEnabled: true,
                    child: listener)),
            Padding(
              padding: const EdgeInsets.symmetric(horizontal: 16),
              child: Row(
                mainAxisAlignment: MainAxisAlignment.start,
                children: [
                  ElevatedButton(
                    child: const Text('Eraser Mode'),
                    style: _getEraserButtonColor(),
                    onPressed: _setEraserMode,
                  )
                ],
              ),
            ),
          ],
        ),
      ),
    );
  }

  void _updatePoints(PointerEvent details) {
    setState(() {
      // Normal write mode
      if (details.kind == ui.PointerDeviceKind.stylus &&
          eraserMode == false) {
        _points = List.from(_points)..add(details.localPosition);
      }
      // Eraser mode
      else if (details.kind == ui.PointerDeviceKind.invertedStylus ||
          (details.kind == ui.PointerDeviceKind.stylus && eraserMode == true)) {
        removePoints(details.localPosition);
      }
      } else {
        // Nur negatives offset addieren
        Offset newOffset = _offset + details.delta;
        double dy = newOffset.dy;
        if (newOffset.dy > 0) {
          dy = 0;
        }
        _offset = Offset(0, dy);
      }
    });
  }

  void removePoints(Offset offset) {
    // check if input offset is close to offsets in _points and return
    // a offset close to input offset if so
    List<Offset> closeOffsets = getClosePoint(offset, _points);
    if (closeOffsets.isNotEmpty) {
      for (var closeOffset in closeOffsets) {
        // get index where stroke starts and ends
        int startIndex = getIndexOfStroke(closeOffset, "start", _points);
        int endIndex = getIndexOfStroke(closeOffset, "end", _points);
        // remove stroke
        setState(() {
          _points.removeRange(startIndex, endIndex + 1);
        });
      }
    }
  }

  List<Offset> getClosePoint(Offset offset, List<Offset?> strokes) {
    List<Offset> closePoints = <Offset>[];
    for (var point in strokes) {
      if (point != null) {
        // calculate distance between input offset and point from _points
        // add point to list if distance is smaler than 10 (the correct radius
        // needs to be evaluated after testing)
        if ((offset - point).distance < 20) {
          closePoints.add(point);
        }
      }
    }
    return closePoints;
  }

  // Returns the first index in _points of the stroke belonging to the input
  // offset. If "end" is passed as location, the last index of the stroke is
  // returned.
  int getIndexOfStroke(Offset offset, String location, List<Offset?> strokes) {
    int index = strokes.indexOf(offset);
    Offset? point = offset;
    int inc = location == "end" ? 1 : -1;
    while (point != null) {
      index += inc;
      if (index == 0) {
        break;
      }
      point = strokes[index];
    }
    return index - inc;
  }

  void _addNull(PointerEvent details) {
      _points.add(null);
  }

  void _clearPad() {
    setState(() {
      _points.clear();
      _recognisedText = '';
      _recognisedOffsetText = '';
    });
  }

  void _setEraserMode() {
    setState(() {
        eraserMode = eraserMode ? false : true;
    });
  }

  ButtonStyle _getEraserButtonColor() {
    Color buttonColor = eraserMode ? Colors.red : Colors.blue;
    return ButtonStyle(
      backgroundColor: MaterialStateProperty.all<Color>(buttonColor),
    );
  }
}

Why is the widget only rebuilding after I hit the Eraser Mode button again? Adding offsets to _points also still works.

Thank you very much in advance!