[ET Trac] #2762: The flesh is not thread safe

[Steven R. Brandt]

#2762: The flesh is not thread safe

 Reporter: Steven R. Brandt
   Status: submitted
Milestone: ET_2023_11
  Version: 
     Type: bug
 Priority: blocker
Component: Cactus

The flesh is not thread safe, and therefore CarpetX’s calling of functions through the flesh via openmp is broken. In particular, the function below The variable :`current_scheduled_function` is a global variable. 

```
const cFunctionData *CCTK_ScheduleQueryCurrentFunction(const cGH * CCTK_ATTRIBUTE_UNUSED GH)
{
  return current_scheduled_function;
}
```

If one makes the following hackish edit \(which I do not recommend\) the problem goes away.

```
db1 flesh$ git diff src/main/ScheduleInterface.c|cat -
diff --git a/src/main/ScheduleInterface.c b/src/main/ScheduleInterface.c
index 93bdc20..c922904 100644
--- a/src/main/ScheduleInterface.c
+++ b/src/main/ScheduleInterface.c
@@ -12,6 +12,7 @@
 #include <stdlib.h>
 #include <stdarg.h>
 #include <string.h>
+#include <assert.h>

 #include "cctk_Flesh.h"
 #include "cctk_WarnLevel.h"
@@ -35,6 +36,8 @@

 #include "util_Table.h"

+#include "omp.h"
+
 static const char *rcsid = "$Header$";

 CCTK_FILEVERSION(main_ScheduleInterface_c);
@@ -238,7 +241,7 @@ static int *scheduled_storage_groups_timelevels = NULL;
 static cTimerData *timerinfo = NULL;
 static int total_timer = -1;

-static const cFunctionData *current_scheduled_function = NULL;
+static const cFunctionData *current_scheduled_function[100];// = NULL;


 /********************************************************************
@@ -286,21 +289,22 @@ int CCTK_CallFunction(void *function,

   int (*oneargfunc)(void *);

+  int tn = omp_get_thread_num();
 #if ALLOW_RECURSIVE_SCHEDULE_CALLS
-  const cFunctionData *previous_scheduled_function = current_scheduled_function;
+  const cFunctionData *previous_scheduled_function = current_scheduled_function[tn];
 #else
-  if(current_scheduled_function != NULL)
+  if(current_scheduled_function[tn] != NULL)
   {
     CCTK_VWarn(CCTK_WARN_PICKY, __LINE__, __FILE__, "Cactus",
                "CCTK_CallFunction: recursive call, calling "
                "'%s: %s::%s' while within '%s: %s::%s'",
                fdata->where, fdata->thorn, fdata->routine,
-               current_scheduled_function->where,
-               current_scheduled_function->thorn,
-               current_scheduled_function->routine);
+               current_scheduled_function[tn]->where,
+               current_scheduled_function[tn]->thorn,
+               current_scheduled_function[tn]->routine);
   }
 #endif
-  current_scheduled_function = fdata;
+  current_scheduled_function[tn] = fdata;

   switch(fdata->type)
   {
@@ -317,6 +321,7 @@ int CCTK_CallFunction(void *function,
       {
         case LangC:
           standardfunc = (void (*)(void *))function;
+          //assert(data != 0);
           standardfunc(data);
           break;
         case LangFortran:
@@ -336,9 +341,9 @@ int CCTK_CallFunction(void *function,
   }

 #if ALLOW_RECURSIVE_SCHEDULE_CALLS
-  current_scheduled_function = previous_scheduled_function;
+  current_scheduled_function[tn] = previous_scheduled_function;
 #else
-  current_scheduled_function = NULL;
+  current_scheduled_function[tn] = NULL;
 #endif

   /* Return 0, meaning didn't synchronise */
@@ -369,7 +374,8 @@ int CCTK_CallFunction(void *function,
 @@*/
 const cFunctionData *CCTK_ScheduleQueryCurrentFunction(const cGH * CCTK_ATTRIBUTE_UNUSED GH)
 {
-  return current_scheduled_function;
+  int tn = omp_get_thread_num();
+  return current_scheduled_function[tn];
 }

 /*@@
```

I think what needs to be done is cctkGH needs to hold the current\_scheduled\_function and CarpetX needs to set it appropriately in its various copies.

--
Ticket URL: https://bitbucket.org/einsteintoolkit/tickets/issues/2762/the-flesh-is-not-thread-safe
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.einsteintoolkit.org/pipermail/trac/attachments/20231108/fbef3bdc/attachment-0001.htm>