aboutsummaryrefslogtreecommitdiff
path: root/libgomp/task.c
diff options
context:
space:
mode:
Diffstat (limited to 'libgomp/task.c')
-rw-r--r--libgomp/task.c168
1 files changed, 167 insertions, 1 deletions
diff --git a/libgomp/task.c b/libgomp/task.c
index 828348c..db4a6f7 100644
--- a/libgomp/task.c
+++ b/libgomp/task.c
@@ -80,6 +80,7 @@ gomp_init_task (struct gomp_task *task, struct gomp_task *parent_task,
task->dependers = NULL;
task->depend_hash = NULL;
task->taskwait = NULL;
+ task->depend_all_memory = NULL;
task->depend_count = 0;
task->completion_sem = NULL;
task->deferred_p = false;
@@ -171,6 +172,7 @@ gomp_task_handle_depend (struct gomp_task *task, struct gomp_task *parent,
size_t ndepend = (uintptr_t) depend[0];
size_t i;
hash_entry_type ent;
+ bool all_memory = false;
if (ndepend)
{
@@ -181,6 +183,7 @@ gomp_task_handle_depend (struct gomp_task *task, struct gomp_task *parent,
{
task->depend[i].addr = depend[2 + i];
task->depend[i].is_in = i >= nout;
+ all_memory |= i < nout && depend[2 + i] == NULL;
}
}
else
@@ -201,6 +204,8 @@ gomp_task_handle_depend (struct gomp_task *task, struct gomp_task *parent,
{
case GOMP_DEPEND_OUT:
case GOMP_DEPEND_INOUT:
+ all_memory |= d[0] == NULL;
+ break;
case GOMP_DEPEND_MUTEXINOUTSET:
break;
case GOMP_DEPEND_IN:
@@ -226,8 +231,126 @@ gomp_task_handle_depend (struct gomp_task *task, struct gomp_task *parent,
task->depend[n++].is_in = 1;
}
}
- task->depend_count = ndepend;
task->num_dependees = 0;
+ if (__builtin_expect (parent->depend_all_memory && ndepend, false))
+ {
+ struct gomp_task *tsk = parent->depend_all_memory;
+ if (tsk->dependers == NULL)
+ {
+ tsk->dependers
+ = gomp_malloc (sizeof (struct gomp_dependers_vec)
+ + 6 * sizeof (struct gomp_task *));
+ tsk->dependers->n_elem = 1;
+ tsk->dependers->allocated = 6;
+ tsk->dependers->elem[0] = task;
+ }
+ else
+ {
+ if (tsk->dependers->n_elem == tsk->dependers->allocated)
+ {
+ tsk->dependers->allocated
+ = tsk->dependers->allocated * 2 + 2;
+ tsk->dependers
+ = gomp_realloc (tsk->dependers,
+ sizeof (struct gomp_dependers_vec)
+ + (tsk->dependers->allocated
+ * sizeof (struct gomp_task *)));
+ }
+ tsk->dependers->elem[tsk->dependers->n_elem++] = task;
+ }
+ task->num_dependees++;
+ }
+ if (__builtin_expect (all_memory, false))
+ {
+ /* A task with depend(inout: omp_all_memory) depends on all previous
+ sibling tasks which have any dependencies and all later sibling
+ tasks which have any dependencies depend on it. */
+ task->depend_count = 1;
+ task->depend[0].addr = NULL;
+ task->depend[0].next = NULL;
+ task->depend[0].prev = NULL;
+ task->depend[0].task = task;
+ task->depend[0].redundant = true;
+ task->depend[0].redundant_out = false;
+ if (parent->depend_hash)
+ {
+ /* Inlined htab_traverse + htab_clear. All newer siblings can
+ just depend on this task. Add dependencies on all previous
+ sibling tasks with dependencies and make them redundant and
+ clear the hash table. */
+ hash_entry_type *slot = &parent->depend_hash->entries[0];
+ hash_entry_type *end = slot + htab_size (parent->depend_hash);
+ for (; slot != end; ++slot)
+ {
+ if (*slot == HTAB_EMPTY_ENTRY)
+ continue;
+ if (*slot != HTAB_DELETED_ENTRY)
+ {
+ for (ent = *slot; ent; ent = ent->next)
+ {
+ struct gomp_task *tsk = ent->task;
+
+ if (ent->redundant_out)
+ break;
+
+ ent->redundant = true;
+ if (tsk->dependers == NULL)
+ {
+ tsk->dependers
+ = gomp_malloc (sizeof (struct gomp_dependers_vec)
+ + 6 * sizeof (struct gomp_task *));
+ tsk->dependers->n_elem = 1;
+ tsk->dependers->allocated = 6;
+ tsk->dependers->elem[0] = task;
+ task->num_dependees++;
+ continue;
+ }
+ /* We already have some other dependency on tsk from
+ earlier depend clause. */
+ else if (tsk->dependers->n_elem
+ && (tsk->dependers->elem[tsk->dependers->n_elem
+ - 1] == task))
+ continue;
+ else if (tsk->dependers->n_elem
+ == tsk->dependers->allocated)
+ {
+ tsk->dependers->allocated
+ = tsk->dependers->allocated * 2 + 2;
+ tsk->dependers
+ = gomp_realloc (tsk->dependers,
+ sizeof (struct gomp_dependers_vec)
+ + (tsk->dependers->allocated
+ * sizeof (struct gomp_task *)));
+ }
+ tsk->dependers->elem[tsk->dependers->n_elem++] = task;
+ task->num_dependees++;
+ }
+ while (ent)
+ {
+ ent->redundant = true;
+ ent = ent->next;
+ }
+ }
+ *slot = HTAB_EMPTY_ENTRY;
+ }
+ if (htab_size (parent->depend_hash) <= 32)
+ {
+ parent->depend_hash->n_elements = 0;
+ parent->depend_hash->n_deleted = 0;
+ }
+ else
+ {
+ /* Shrink the hash table if it would be too large.
+ We don't want to walk e.g. megabytes of empty hash
+ table for every depend(inout: omp_all_memory). */
+ free (parent->depend_hash);
+ parent->depend_hash = htab_create (12);
+ }
+ }
+ parent->depend_all_memory = task;
+ return;
+ }
+ task->depend_count = ndepend;
if (parent->depend_hash == NULL)
parent->depend_hash = htab_create (2 * ndepend > 12 ? 2 * ndepend : 12);
for (i = 0; i < ndepend; i++)
@@ -1175,6 +1298,8 @@ gomp_task_run_post_handle_depend_hash (struct gomp_task *child_task)
struct gomp_task *parent = child_task->parent;
size_t i;
+ if (parent->depend_all_memory == child_task)
+ parent->depend_all_memory = NULL;
for (i = 0; i < child_task->depend_count; i++)
if (!child_task->depend[i].redundant)
{
@@ -1738,6 +1863,17 @@ gomp_task_maybe_wait_for_dependencies (void **depend)
n = 5;
}
gomp_mutex_lock (&team->task_lock);
+ if (__builtin_expect (task->depend_all_memory && ndepend, false))
+ {
+ struct gomp_task *tsk = task->depend_all_memory;
+ if (!tsk->parent_depends_on)
+ {
+ tsk->parent_depends_on = true;
+ ++num_awaited;
+ if (tsk->num_dependees == 0 && tsk->kind == GOMP_TASK_WAITING)
+ priority_queue_upgrade_task (tsk, task);
+ }
+ }
for (i = 0; i < ndepend; i++)
{
elem.addr = depend[i + n];
@@ -1760,6 +1896,36 @@ gomp_task_maybe_wait_for_dependencies (void **depend)
}
elem.addr = d[0];
}
+ if (__builtin_expect (elem.addr == NULL && !elem.is_in, false))
+ {
+ size_t size = htab_size (task->depend_hash);
+ if (htab_elements (task->depend_hash) * 8 < size && size > 32)
+ htab_expand (task->depend_hash);
+
+ /* depend(inout: omp_all_memory) - depend on all previous
+ sibling tasks that do have dependencies. Inlined
+ htab_traverse. */
+ hash_entry_type *slot = &task->depend_hash->entries[0];
+ hash_entry_type *end = slot + htab_size (task->depend_hash);
+ for (; slot != end; ++slot)
+ {
+ if (*slot == HTAB_EMPTY_ENTRY || *slot == HTAB_DELETED_ENTRY)
+ continue;
+ for (ent = *slot; ent; ent = ent->next)
+ {
+ struct gomp_task *tsk = ent->task;
+ if (!tsk->parent_depends_on)
+ {
+ tsk->parent_depends_on = true;
+ ++num_awaited;
+ if (tsk->num_dependees == 0
+ && tsk->kind == GOMP_TASK_WAITING)
+ priority_queue_upgrade_task (tsk, task);
+ }
+ }
+ }
+ break;
+ }
ent = htab_find (task->depend_hash, &elem);
for (; ent; ent = ent->next)
if (elem.is_in && ent->is_in)