event listener's event is being used more than once

Question

I have this weird bug that whenever I want to add an item/card to the list and I cycle between clicking the text input and then clicking outside of it, the list won't add only one item but the number of times I pressed in and out.

const main = document.querySelector('main')

main.addEventListener('click', (e) => {

  // Name a group

  if (e.target.id === 'groupNameInput') {
    console.log(e.target)
    let groupNameInput = e.target
    groupNameInput.addEventListener('change', () => {
      let groupName = groupNameInput.parentElement
      console.log(groupNameInput.value)
      groupNameInput.style.display = 'none'
      groupName.innerText = `${groupNameInput.value}`
      groupName.parentElement.lastElementChild.classList.remove('notAssigned')
      groupName.parentElement.lastElementChild.style.display = 'inline'
    })
  } else if (e.target.classList == 'addCardInput' && e.target.classList != 'notAssigned') {

    let addCardInput = e.target
    let taskUl = addCardInput.parentElement.querySelector('.taskUl')
    addCardInput.addEventListener("keydown", function(event) {
      if (event.key == 'Enter') {
        console.log("I'm in!")
        let newLi = document.createElement('li')
        newLi.innerHTML = `${event.target.value}`
        taskUl.appendChild(newLi)
      }
    })
  }
})

<main>
  <section class="groupSection">
    <span class="groupName">
                        <input type="text" autocomplete="false" id="groupNameInput" placeholder="name">
                    </span>
    <ul class="taskUl">
    </ul>
    <input type="text" class="addCardInput" placeholder="Add a card...">
  </section>
  <section class="addItemsSection">
    <button id="addItemsBtn">
                        <img src="./srcs/add_Icon.png" alt="" id="addItemsImage">
                    </button>
  </section>
</main>