Problem
Here’s my updated version of the problem I posted at Calculating time windows for entities, adding in suggested changes as well as a change I did so that the Reflection is ‘cached’ in a ConcurrentDictionary and happens only once per Type.
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Reflection;
namespace Services.Helpers
{
#region Custom Attributes
[AttributeUsage(AttributeTargets.Property)]
public class DoNotCopyIntoTimeWindow : Attribute { } // leave default
[AttributeUsage(AttributeTargets.Property)]
public class IsProcessedIntoTimeWindow : Attribute { } // calculate time window for this property
[AttributeUsage(AttributeTargets.Property)]
public class IsTimeWindowDate : Attribute { } // attribute to mark property as the datetime
[AttributeUsage(AttributeTargets.Property)]
public class IsTimeWindowIdentifier : Attribute { } // this is the time window property
#endregion
public class TimeWindow
{
#region Structs
public struct TimeWindowDictionary
{
public PropertyInfo PropertyInfo { get; set; }
public Dictionary<NullObject<dynamic>, int> Dictionary { get; set; }
}
public struct NullObject<T>
{
[DefaultValue(true)]
private readonly bool isnull; // default property initializers are not supported for structs
private NullObject(T item, bool isnull) : this()
{
this.isnull = isnull;
Item = item;
}
public NullObject(T item) : this(item, item == null)
{
}
public static NullObject<T> Null()
{
return new NullObject<T>();
}
public T Item { get; private set; }
public bool IsNull()
{
return isnull;
}
public static implicit operator T(NullObject<T> nullObject)
{
return nullObject.Item;
}
public static implicit operator NullObject<T>(T item)
{
return new NullObject<T>(item);
}
public override string ToString()
{
return (Item != null) ? Item.ToString() : "NULL";
}
public override bool Equals(object obj)
{
if (obj == null)
return IsNull();
if (!(obj is NullObject<T>))
return false;
var no = (NullObject<T>)obj;
if (IsNull())
return no.IsNull();
if (no.IsNull())
return false;
return Item.Equals(no.Item);
}
public override int GetHashCode()
{
if (IsNull())
return 0;
var result = Item.GetHashCode();
if (result >= 0)
result++;
return result;
}
}
public struct Properties
{
public List<PropertyInfo> PropertiesToProcess { get; set; }
public List<PropertyInfo> CopyProperties { get; set; }
public PropertyInfo TimeWindowIdentifier { get; set; }
public PropertyInfo DatePropertyInfo { get; set; }
public int Size { get; set; }
}
#endregion
#region Class Members
private static readonly ConcurrentDictionary<Type, Properties> PropertiesDictionary = new ConcurrentDictionary<Type, Properties>();
#endregion
#region Methods
public static IEnumerable<T> CalculateTimeWindows<T>(DateTime dateFrom, DateTime dateTo, List<T> stateModels) where T : new()
{
if (stateModels.Count() == 0)
return new List<T>();
dateFrom = GetPropertiesAndDictionaries(
dateFrom,
stateModels,
out PropertyInfo datePropertyInfo,
out List<PropertyInfo> copyProperties,
out PropertyInfo timeWindowIdentifier,
out int size,
out TimeWindowDictionary[] dictionaries);
byte[] windowDurations = { 5, 15, 60 };
return windowDurations.SelectMany(wd =>
CalculateTimeWindow(
dateFrom,
dateTo,
stateModels,
wd,
datePropertyInfo,
copyProperties,
timeWindowIdentifier,
size,
dictionaries));
}
public static IEnumerable<T> CalculateTimeWindow<T>(DateTime dateFrom, DateTime dateTo, List<T> stateModels, byte timeWindowMinutes, PropertyInfo datePropertyInfo, List<PropertyInfo> copyProperties, PropertyInfo timeWindowIdentifier, int size, TimeWindowDictionary[] dictionaries) where T : new()
{
if (stateModels.Count() > 0)
{
DateTime currentWindowFrom, currentWindowTo, nextWindowFrom;
nextWindowFrom = dateFrom;
int itemPointer = 0;
T prevItem = default;
T prevTimeWindow = default;
int j = 1;
do // one time window
{
for (int i = 0; i < size; i++)
dictionaries[i].Dictionary = new Dictionary<NullObject<dynamic>, int>();
currentWindowFrom = nextWindowFrom;
nextWindowFrom = currentWindowFrom.AddMinutes(timeWindowMinutes);
currentWindowTo = nextWindowFrom.AddSeconds(-1);
var calculateTime = currentWindowFrom;
for (; itemPointer < stateModels.Count(); itemPointer++)
{
var item = stateModels.ElementAt(itemPointer);
var date = (DateTime)datePropertyInfo.GetValue(item);
if (date >= currentWindowTo)
break;
var endDate = (date > currentWindowTo) ? nextWindowFrom : date; // state might extend more than the end of the time window
CalculateStateSeconds(prevItem, dictionaries, calculateTime, endDate);
prevItem = item;
calculateTime = (date < currentWindowFrom) ? currentWindowFrom : date; // to fix the 'yesterday' date
}
if (calculateTime < currentWindowTo)
CalculateStateSeconds(prevItem, dictionaries, calculateTime, nextWindowFrom);
if (dictionaries[0].Dictionary.Count > 0)
{
bool sameAsPrevious = (prevTimeWindow != null);
var output = new T();
foreach (var dictionary in dictionaries)
{
var maxValue = dictionary.Dictionary.First();
for (j = 1; j < dictionary.Dictionary.Count; j++)
{
var valuePair = dictionary.Dictionary.ElementAt(j);
if (valuePair.Value > maxValue.Value)
maxValue = valuePair;
}
var valToSet = maxValue.Key.Item;
if (sameAsPrevious)
{
var prevVal = GetValue(prevTimeWindow, dictionary.PropertyInfo);
if (!(valToSet == null && prevVal == null))
sameAsPrevious = (valToSet == prevVal);
}
SetValue(output, dictionary.PropertyInfo, valToSet);
}
if (!sameAsPrevious)
{
foreach (var copyProperty in copyProperties)
SetValue(output, copyProperty, copyProperty.GetValue(prevItem));
timeWindowIdentifier.SetValue(output, timeWindowMinutes);
datePropertyInfo.SetValue(output, currentWindowFrom);
prevTimeWindow = output;
yield return output;
}
}
}
while (nextWindowFrom <= dateTo);
}
}
private static DateTime GetPropertiesAndDictionaries<T>(DateTime dateFrom, List<T> stateModels, out PropertyInfo datePropertyInfo, out List<PropertyInfo> copyProperties, out PropertyInfo timeWindowIdentifier, out int size, out TimeWindowDictionary[] dictionaries) where T : new()
{
Type tType = typeof(T);
if (!PropertiesDictionary.TryGetValue(tType, out Properties properties))
{
var propInfos = tType.GetProperties();
datePropertyInfo = propInfos.Single(p => p.GetCustomAttributes(typeof(IsTimeWindowDate), true).Any());
var propertiesToProcess = propInfos.Where(p => p.GetCustomAttributes(typeof(IsProcessedIntoTimeWindow), true).Any()).ToList();
copyProperties = propInfos.Where(p => !p.GetCustomAttributes(typeof(IsTimeWindowIdentifier), true).Any() && !p.GetCustomAttributes(typeof(DoNotCopyIntoTimeWindow), true).Any() && !p.GetCustomAttributes(typeof(IsTimeWindowDate), true).Any() && !p.GetCustomAttributes(typeof(IsProcessedIntoTimeWindow), true).Any() && p.CanWrite && !p.GetMethod.IsVirtual).ToList();
timeWindowIdentifier = propInfos.Single(p => p.GetCustomAttributes(typeof(IsTimeWindowIdentifier), true).Any());
size = propertiesToProcess.Count();
properties = new Properties()
{
CopyProperties = copyProperties,
DatePropertyInfo = datePropertyInfo,
PropertiesToProcess = propertiesToProcess,
TimeWindowIdentifier = timeWindowIdentifier,
Size = size
};
PropertiesDictionary.TryAdd(tType, properties);
}
else
{
datePropertyInfo = properties.DatePropertyInfo;
copyProperties = properties.CopyProperties;
timeWindowIdentifier = properties.TimeWindowIdentifier;
size = properties.Size;
}
dictionaries = properties.PropertiesToProcess
.Select(p => new TimeWindowDictionary { PropertyInfo = p })
.ToArray();
var firstDate = (DateTime)datePropertyInfo.GetValue(stateModels.First());
if (firstDate < dateFrom)
dateFrom = new DateTime(firstDate.Year, firstDate.Month, firstDate.Day, firstDate.Hour, 0, 0, DateTimeKind.Utc);
return dateFrom;
}
private static dynamic GetValue(object inputObject, PropertyInfo propertyInfo)
{
return propertyInfo.GetValue(inputObject);
}
//private static void SetValue(object inputObject, string propertyName, object propertyVal)
private static void SetValue(object inputObject, PropertyInfo propertyInfo, object propertyVal)
{
if (propertyVal != null)
{
//find the property type
Type propertyType = propertyInfo.PropertyType;
//Convert.ChangeType does not handle conversion to nullable types
//if the property type is nullable, we need to get the underlying type of the property
var targetType = IsNullableType(propertyType) ? Nullable.GetUnderlyingType(propertyType) : propertyType;
//Returns an System.Object with the specified System.Type and whose value is
//equivalent to the specified object.
propertyVal = Convert.ChangeType(propertyVal, targetType);
}
//Set the value of the property
propertyInfo.SetValue(inputObject, propertyVal, null);
}
private static bool IsNullableType(Type type)
{
return type.IsGenericType && type.GetGenericTypeDefinition().Equals(typeof(Nullable<>));
}
private static void CalculateStateSeconds<T>(T prevItem, IEnumerable<TimeWindowDictionary> dictionaries, DateTime calculateTime, DateTime endDate)
{
if (prevItem != null)
{
var seconds = Convert.ToInt32(endDate.Subtract(calculateTime).TotalSeconds);
foreach (var dictionary in dictionaries)
{
var key = dictionary.PropertyInfo.GetValue(prevItem);
dictionary.Dictionary.TryGetValue(key, out int existingSeconds);
dictionary.Dictionary[key] = existingSeconds + seconds;
}
}
}
#endregion
}
}
Now the problem I’m having is that when I call the method in this way:
var stopWatchTW = new Stopwatch();
stopWatchTW.Start();
CalculateTimeWindows();
stopWatchTW.Stop();
ConsoleLogger.WriteLine($"Processing time windows took {stopWatchTW.ElapsedMilliseconds}ms");
private void CalculateTimeWindows()
{
myList1.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList1).ToList());
myList2.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList2).ToList());
myList3.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList3).ToList());
myList4.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList4).ToList());
myList5.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList5).ToList());
myList6.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList6).ToList());
myList7.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList7).ToList());
myList8.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList8).ToList());
myList9.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList9).ToList());
myList10.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList10).ToList());
myList11.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList11).ToList());
myList12.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList12).ToList());
myList13.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList13).ToList());
myList14.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList14).ToList());
myList15.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList15).ToList());
myList16.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList16).ToList());
}
where each myList would be an empty list, the code is taking a very long time to run when deployed as an Azure WebJob (4.5 to 8 SECONDS to run), despite that I have this at the very top of the CalculateTimeWindows method:
if (stateModels.Count() == 0)
return new List<T>();
Solution
Well, this is embarrassing. Turns out, that the lists weren’t empty after all, but they had 1 item, with its date property (having the attribute IsTimeWindowDate) preceding the dateFrom. The result of CalculateTimeWindows would still be an empty list, and at the end I was only counting the rows inserted in the database (taking for example myList1.Where(x => x.Date >= dateFrom)).
Still, this is taking a wee bit too much for 1 row.
EDIT: An improvement with this code whilst debugging locally, but actually taking longer when deployed live on Azure:
(first setting lastWindow = false; inside the CalculateTimeWindows)
...
if (itemPointer == stateModels.Count())
lastWindow = true;
else if (lastWindow)
break;
}
while (nextWindowFrom <= dateTo);