How can I restrict who can make changes using interfaces?

Question

I have a IHasTag interface, a TaggableItem class that implements IHasTag and a TaggingManager class which I want to make it the only one responsible for setting or clearing the Tags but I've been struggling all day trying to implement but with no luck. How can I make this possible?

public class TaggableItem : TaggingManager.IHasTag
{
    public string Tag { get; }  // read only
}

public class TaggingManager
{
    public interface IHasTag
    {
        string Tag { get; }
    }

    private List<IHasTag> _taggedItems = new();

    public void TagItem(IHasTag item, string tag)
    {
        item.Tag = tag;  // not working
        _taggedItems.Add(item);
    }

    public void ClearAllTags()
    {
        foreach (var item in _taggedItems)
        {
            item.Tag = "":  // not working
        }
        _taggedItems.Clear();
    }
}

EDIT I followed Thomas' suggestion and this is what I end up doing. I know it's not perfect, though. Thank you all for your advices.

public interface ITaggable
{
    string? Tag { get; }
}

public interface ISelectable
{
    bool IsSelected { get; }
}

public interface IItem : ITaggable, ISelectable
{
}

public class Item : IItem
{
    protected Item() { }
    public bool IsSelected { get; set; }
    public string Tag { get; set; } = string.Empty;
    // 'Item' will be created here returning IItem.
    // So, unless you 'cast' it, you can't set 'Tag' or 'IsSelected'.
    public static IItem CreateItem() => new Item();
}

public class SelectionManager
{
    protected List<object> _items = new();
    public void Select(ISelectable item)
    {
        if (item is Item selectable)
        {
            selectable.IsSelected = true;
            _items.Add(item);
        }
    }
    public void Unselect(ISelectable item)
    {
        if (item is Item selectable)
        {
            selectable.IsSelected = false;
            _items.Remove(item);
        }
    }
}

public class TaggingManager
{
    private List<object> _items = new();
    public void Tag(ITaggable item, string tag)
    {
        if (item is Item taggable)
        {
            taggable.Tag = tag;
            _items.Add(item);
        }
    }
    public void Untag(ITaggable item)
    {
        if (item is Item taggable)
        {
            taggable.Tag = string.Empty;
            _items.Remove(item);
        }
    }
}

Answer 1

One possible option is to move the "getter" also to the TagManager. Then the manager is responsible for the tags. The object itself does not even need to know about the tags.

You still can restrict this by exchanging object with an interface.

public class TagManager
{
    private Dictionary<object, string> _tagedItems = new Dictionary<object, string>();

    public bool HasTag(object item)
    {
       return _tagedItems.ContainsKey(item);
    }

    public string GetTag(object item)
    {
      return  _tagedItems[item];
    }

    public void SetTag(object item, string tag)
    {
        if(!HasTag(item))
        {
            _tagedItems.Add(item, tag);
        }
        else
        {
            _tagedItems[item] = tag;
        }        
    }
}

I don't think that this is really a answer to the OP, but one possible solution for the underlying problem.