Maximize/Show an already open window when button is clicked

Question

I have a main application that through button presses allows other application to open. If a certain sub application is already open, the click on that specific button or shortkey should bring that sub application to the foreground.

As of now, ShowWindow(hwnd, SW_SHOW), ShowWindow(hwnd, SW_MAXIMIZE), SetFocus(hwnd), SetForegroundWindow(hwnd) show no effect.

HWND myhwnd;

BOOL CALLBACK FindCustomWindow(HWND hwnd, LPARAM)
{
   char buffer[101];
   buffer[100]=0;
   GetWindowText(hwnd, buffer, 100);
   if(strstr(buffer,(const char *)"Custom Sub App Name -")!=NULL)
   {
      myhwnd=hwnd;
      return(false);
   }
   return(true);
}
//---------------------------------------------------------------------------

CUSTOM_ENTRY_WINMAIN (custom);

static int custom_WinMain(HINSTANCE hInstance, HINSTANCE, LPSTR lpszCmdLine, int)
{
   if (!net::init (HInstance))
   {
      LOG_ERROR (err_get());
   }
   int retval=0;

   HANDLE hMutex;
   hMutex=CreateMutex(NULL,true,"custom sub app");
   if(GetLastError()==ERROR_ALREADY_EXISTS)
   {
      myhwnd=NULL;
      EnumWindows((WNDENUMPROC)FindCustomWindow, 0);
      if(myhwnd!=NULL)
      {
         HANDLE mem=CreateFileMapping((void *)0xFFFFFFFF, NULL, PAGE_READWRITE, 0, strlen(lpszCmdLine)+1, "custommem");
         if (mem != NULL)
         {
            ShowMessage("mem != NULL");
            char *test=(char *)MapViewOfFile(mem, FILE_MAP_WRITE, 0, 0, strlen(lpszCmdLine)+1);
            strcpy(test, lpszCmdLine);
            ::SendMessage(myhwnd, WM_OPENCUSTOM, 0, strlen(lpszCmdLine)+1);
            UnmapViewOfFile(test);
            CloseHandle(mem);
         }
      }
   }
   else
   {

      if((retval=Start(lpszCmdLine, hMutex))==0)
      {
         OSVERSIONINFO os;
         os.dwOSVersionInfoSize=sizeof(OSVERSIONINFO);
         GetVersionEx(&os);
         if(os.dwMajorVersion==5)
         {
            typedef void (WINAPI *tSetThemeAppProperties)(  DWORD );

            tSetThemeAppProperties pSetThemeAppProperties=0;
            HINSTANCE  handle  = ::LoadLibraryA("UxTheme.dll");

            if (handle)
               pSetThemeAppProperties = (tSetThemeAppProperties) ::GetProcAddress(handle,"SetThemeAppProperties");

            if ( pSetThemeAppProperties)
            {
               pSetThemeAppProperties(0);
            }
            ::FreeLibrary(handle);
         }
         Application->Initialize();
         Application->Title = "Custom Sub App Name";
         Application->CreateForm(__classid(TfrmCustom), &frmCustom);
         Application->Run();
         frmCustom->cleanup();
      }
   }
   if(hMutex!=0)
      CloseHandle(hMutex);
   net::shutdown();
   return(retval);
} 

As you can see, it prevents another instance of that sub app to open, via mutex. Although, funnily enough for me, when I implement ShowWindow(...) after the mutex:

HANDLE hMutex;
hMutex=CreateMutex(NULL,true,"custom sub app");
ShowWindow(myhwnd, SW_SHOW);
if(GetLastError()==ERROR_ALREADY_EXISTS)
{ ...

I can somehow surpass the mutex and create as many new instances of the sub app as I want. The if-condition fails.

I am fairly new to programming, the code I'm working with is really old stuff and relies heavily on a lot of other custom-made parts which I unfortunately can't provide here, so I am now quite lost on how to proceed.

Am I somehow able to cleverly utilize Handles to bring the already open sub app to front, or is a wholey other solution necessary?