#2762: The flesh is not thread safe - Trac

8 Nov 2023

#2762: The flesh is not thread safe
Reporter: Steven R. Brandt
   Status: submitted
Milestone: ET_2023_11
  Version: 
     Type: bug
 Priority: blocker
Component: Cactus
The flesh is not thread safe, and therefore CarpetX’s calling of functions through the flesh via openmp is broken. In particular, the function below The variable :`current_scheduled_function` is a global variable.
```
const cFunctionData *CCTK_ScheduleQueryCurrentFunction(const cGH * CCTK_ATTRIBUTE_UNUSED GH)
{
  return current_scheduled_function;
}
```
If one makes the following hackish edit (which I do not recommend) the problem goes away.
```
db1 flesh$ git diff src/main/ScheduleInterface.c|cat -

diff --git a/src/main/ScheduleInterface.c b/src/main/ScheduleInterface.c
index 93bdc20..c922904 100644
--- a/src/main/ScheduleInterface.c
+++ b/src/main/ScheduleInterface.c
@@ -12,6 +12,7 @@
 #include <stdlib.h>
 #include <stdarg.h>
 #include <string.h>
+#include <assert.h>
#include "cctk_Flesh.h"
 #include "cctk_WarnLevel.h"
@@ -35,6 +36,8 @@
#include "util_Table.h"
+#include "omp.h"
+
 static const char *rcsid = "$Header$";
CCTK_FILEVERSION(main_ScheduleInterface_c);
@@ -238,7 +241,7 @@ static int *scheduled_storage_groups_timelevels = NULL;
 static cTimerData *timerinfo = NULL;
 static int total_timer = -1;
-static const cFunctionData *current_scheduled_function = NULL;
+static const cFunctionData *current_scheduled_function[100];// = NULL;
/********************************************************************
@@ -286,21 +289,22 @@ int CCTK_CallFunction(void *function,
int (*oneargfunc)(void *);
+  int tn = omp_get_thread_num();
 #if ALLOW_RECURSIVE_SCHEDULE_CALLS
-  const cFunctionData *previous_scheduled_function = current_scheduled_function;
+  const cFunctionData *previous_scheduled_function = current_scheduled_function[tn];
 #else
-  if(current_scheduled_function != NULL)
+  if(current_scheduled_function[tn] != NULL)
   {
     CCTK_VWarn(CCTK_WARN_PICKY, __LINE__, __FILE__, "Cactus",
                "CCTK_CallFunction: recursive call, calling "
                "'%s: %s::%s' while within '%s: %s::%s'",
                fdata->where, fdata->thorn, fdata->routine,
-               current_scheduled_function->where,
-               current_scheduled_function->thorn,
-               current_scheduled_function->routine);
+               current_scheduled_function[tn]->where,
+               current_scheduled_function[tn]->thorn,
+               current_scheduled_function[tn]->routine);
   }
 #endif
-  current_scheduled_function = fdata;
+  current_scheduled_function[tn] = fdata;
switch(fdata->type)
   {
@@ -317,6 +321,7 @@ int CCTK_CallFunction(void *function,
       {
         case LangC:
           standardfunc = (void (*)(void *))function;
+          //assert(data != 0);
           standardfunc(data);
           break;
         case LangFortran:
@@ -336,9 +341,9 @@ int CCTK_CallFunction(void *function,
   }
#if ALLOW_RECURSIVE_SCHEDULE_CALLS
-  current_scheduled_function = previous_scheduled_function;
+  current_scheduled_function[tn] = previous_scheduled_function;
 #else
-  current_scheduled_function = NULL;
+  current_scheduled_function[tn] = NULL;
 #endif
/* Return 0, meaning didn't synchronise */
@@ -369,7 +374,8 @@ int CCTK_CallFunction(void *function,
 @@*/
 const cFunctionData *CCTK_ScheduleQueryCurrentFunction(const cGH * CCTK_ATTRIBUTE_UNUSED GH)
 {
-  return current_scheduled_function;
+  int tn = omp_get_thread_num();
+  return current_scheduled_function[tn];
 }
/*@@
```
I think what needs to be done is cctkGH needs to hold the current_scheduled_function and CarpetX needs to set it appropriately in its various copies.
--
Ticket URL: https://bitbucket.org/einsteintoolkit/tickets/issues/2762/the-flesh-is-not-t...

    