Iterating Big SharePoint Lists In Small Chunks

Why would anyone want to iterate SharePoint lists in small chunks? because of two main reasons, the first one is because if the results of one of your queries is too big (say 2000 items) it gets really slow to work with it and second because you might just need the top 20 items that fulfil a condition and that condition is too complex to be represented in the first CAML query.

I am using query pagination to do it and it’s about 75% faster than processing all of the items in the first query (this depends on the amount of data you have in the list)

It basically goes as follows, you set the RowLimit to a sensible number (you also want to keep the number of round trips to a minimum) then you query the SharePoint List until the number of items that fulfil the condition hits the target or you run out of items. (I still haven’t tested if it’s faster to go through every item in a big list using this method or retrieving all at once but something tells me this will be faster)

/// <summary>
/// Returns the top MaxResults items in the query
/// </summary>
internal static List<SPListItem> GetTopItems(SPWeb web, uint MaxResults = 0)
{
    SPList ListToQuery = web.Lists.GetListByInternalName("ListInternalName");


    List<SPListItem> result = new List<SPListItem>();

    SPQuery query = new SPQuery();
    query.Query = string.Format(@"<Where><Eq><FieldRef Name='State' /><Value Type='Choice'>Live</Value></Eq></Where>");
    query.ViewAttributes = "Scope='RecursiveAll'";
    query.RowLimit = MaxResults + 1;

    SPListItemCollection QueryResults;
    string PagingInfo = string.Empty;

    while (result.Count < MaxResults && PagingInfo != null)
    {
        //We set the beginning of the query to the last item in the previous page
        query.ListItemCollectionPosition = new SPListItemCollectionPosition(PagingInfo);

        //We get the page of items
        QueryResults = ListToQuery.GetItems(query);

        foreach (SPListItem item in QueryResults)
        {
            //We process the items and add them to the result list
            if (ComplexProcessing(item))
                result.Add(item);
        }

        //We set PagingInfo to the last item retrieved in the query
        if (QueryResults.ListItemCollectionPosition != null)
            PagingInfo = QueryResults.ListItemCollectionPosition.PagingInfo.ToStringSafe();
        else //PagingInfo will be null if we reach the end of the pagination
            PagingInfo = null;
    }

    return result;
}

This approach to paginated CAML queries have improved the performance of my site hugely, I hope it serves you as well.

ObservableDictionary for Silverlight

I have reused copied the code from Shimmy's Blog but had to tweak it a bit to make it fit for Silverlight. At the moment I have used it just in one of my projects but works very well.

using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.ComponentModel;
using System.Linq;

namespace SLUtils.Classes
{
        public class ObservableDictionary<TKey, TValue> : IDictionary<TKey, TValue>, INotifyCollectionChanged, INotifyPropertyChanged
        {
            private const string CountString = "Count";
            private const string IndexerName = "Item[]";
            private const string KeysName = "Keys";
            private const string ValuesName = "Values";

            private IDictionary<TKey, TValue> _Dictionary;
            protected IDictionary<TKey, TValue> Dictionary
            {
                get { return _Dictionary; }
            }

            #region Constructors
            public ObservableDictionary()
            {
                _Dictionary = new Dictionary<TKey, TValue>();
            }
            public ObservableDictionary(IDictionary<TKey, TValue> dictionary)
            {
                _Dictionary = new Dictionary<TKey, TValue>(dictionary);
            }
            public ObservableDictionary(IEqualityComparer<TKey> comparer)
            {
                _Dictionary = new Dictionary<TKey, TValue>(comparer);
            }
            public ObservableDictionary(int capacity)
            {
                _Dictionary = new Dictionary<TKey, TValue>(capacity);
            }
            public ObservableDictionary(IDictionary<TKey, TValue> dictionary, IEqualityComparer<TKey> comparer)
            {
                _Dictionary = new Dictionary<TKey, TValue>(dictionary, comparer);
            }
            public ObservableDictionary(int capacity, IEqualityComparer<TKey> comparer)
            {
                _Dictionary = new Dictionary<TKey, TValue>(capacity, comparer);
            }
            #endregion

            #region IDictionary<TKey,TValue> Members

            public void Add(TKey key, TValue value)
            {
                Insert(key, value, true);
            }

            public bool ContainsKey(TKey key)
            {
                return Dictionary.ContainsKey(key);
            }

            public ICollection<TKey> Keys
            {
                get { return Dictionary.Keys; }
            }

            public bool Remove(TKey key)
            {
                if (key == null) throw new ArgumentNullException("key");

                TValue value;
                Dictionary.TryGetValue(key, out value);

                int index = GetIndex(Dictionary, key);
                var removed = Dictionary.Remove(key);

                if (removed)
                    OnCollectionChanged(NotifyCollectionChangedAction.Remove, new KeyValuePair<TKey, TValue>(key, value), index);
                    //OnCollectionChanged();
                return removed;
            }

            public bool TryGetValue(TKey key, out TValue value)
            {
                return Dictionary.TryGetValue(key, out value);
            }

            public ICollection<TValue> Values
            {
                get { return Dictionary.Values; }
            }

            public TValue this[TKey key]
            {
                get
                {
                    return Dictionary[key];
                }
                set
                {
                    Insert(key, value, false);
                }
            }

            #endregion

            #region ICollection<KeyValuePair<TKey,TValue>> Members

            public void Add(KeyValuePair<TKey, TValue> item)
            {
                Insert(item.Key, item.Value, true);
            }

            public void Clear()
            {
                if (Dictionary.Count > 0)
                {
                    Dictionary.Clear();
                    OnCollectionChanged();
                }
            }

            public bool Contains(KeyValuePair<TKey, TValue> item)
            {
                return Dictionary.Contains(item);
            }

            public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
            {
                Dictionary.CopyTo(array, arrayIndex);
            }

            public int Count
            {
                get { return Dictionary.Count; }
            }

            public bool IsReadOnly
            {
                get { return Dictionary.IsReadOnly; }
            }

            public bool Remove(KeyValuePair<TKey, TValue> item)
            {
                return Remove(item.Key);
            }

            #endregion

            #region IEnumerable<KeyValuePair<TKey,TValue>> Members

            public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
            {
                return Dictionary.GetEnumerator();
            }

            #endregion

            #region IEnumerable Members

            IEnumerator IEnumerable.GetEnumerator()
            {
                return ((IEnumerable)Dictionary).GetEnumerator();
            }

            #endregion

            #region INotifyCollectionChanged Members

            public event NotifyCollectionChangedEventHandler CollectionChanged;

            #endregion

            #region INotifyPropertyChanged Members

            public event PropertyChangedEventHandler PropertyChanged;

            #endregion

            public void AddRange(IDictionary<TKey, TValue> items)
            {
                if (items == null) throw new ArgumentNullException("items");

                if (items.Count > 0)
                {
                    if (Dictionary.Count > 0)
                    {
                        if (items.Keys.Any((k) => Dictionary.ContainsKey(k)))
                            throw new ArgumentException("An item with the same key has already been added.");
                        else
                            foreach (var item in items) Dictionary.Add(item);
                    }
                    else
                        _Dictionary = new Dictionary<TKey, TValue>(items);

                    OnCollectionChanged(NotifyCollectionChangedAction.Add, items.ToArray(), GetIndex(Dictionary, items.Keys.First())); //We notify the index of the first added key
                }
            }

            private void Insert(TKey key, TValue value, bool add)
            {
                if (key == null) throw new ArgumentNullException("key");

                TValue item;
                if (Dictionary.TryGetValue(key, out item))
                {
                    if (add) throw new ArgumentException("An item with the same key has already been added.");
                    if (Equals(item, value)) return;
                    Dictionary[key] = value;

                    OnCollectionChanged(NotifyCollectionChangedAction.Replace, new KeyValuePair<TKey, TValue>(key, value), new KeyValuePair<TKey, TValue>(key, item), GetIndex(Dictionary, key));
                }
                else
                {
                    Dictionary[key] = value;

                    OnCollectionChanged(NotifyCollectionChangedAction.Add, new KeyValuePair<TKey, TValue>(key, value), GetIndex(Dictionary, key));
                }
            }

            private void OnPropertyChanged()
            {
                OnPropertyChanged(CountString);
                OnPropertyChanged(IndexerName);
                OnPropertyChanged(KeysName);
                OnPropertyChanged(ValuesName);
            }

            private int GetIndex(IDictionary<TKey, TValue> Dictionary, TKey key)
            {
                int i = 0;
                foreach (var dictKey in Dictionary.Keys)
                {
                    if (dictKey.Equals(key)) return i;
                }

                return -1;
            }

            protected virtual void OnPropertyChanged(string propertyName)
            {
                if (PropertyChanged != null) PropertyChanged(this, new PropertyChangedEventArgs(propertyName));
            }

            private void OnCollectionChanged()
            {
                OnPropertyChanged();
                if (CollectionChanged != null) CollectionChanged(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset));
            }

            private void OnCollectionChanged(NotifyCollectionChangedAction action, KeyValuePair<TKey, TValue> changedItem, int index)
            {
                OnPropertyChanged();
                if (CollectionChanged != null) CollectionChanged(this, new NotifyCollectionChangedEventArgs(action, changedItem, index));
            }

            private void OnCollectionChanged(NotifyCollectionChangedAction action, KeyValuePair<TKey, TValue> newItem, KeyValuePair<TKey, TValue> oldItem, int index)
            {
                OnPropertyChanged();
                if (CollectionChanged != null) CollectionChanged(this, new NotifyCollectionChangedEventArgs(action, newItem, oldItem, index));
            }

            private void OnCollectionChanged(NotifyCollectionChangedAction action, IList newItems, int index)
            {
                OnPropertyChanged();
                if (CollectionChanged != null) CollectionChanged(this, new NotifyCollectionChangedEventArgs(action, newItems, index));
            }

            public override string ToString()
            {
                return string.Format("ObservableDictionary<{0}> Count={1}", GetKeyAndValue(Dictionary.GetType().GetGenericArguments()), Count);
            }

            private object GetKeyAndValue(Type[] Types)
            {
                string result = string.Empty;

                foreach (Type type in Types)
                {
                    result += type.ToString() + ",";
                }

                if (!string.IsNullOrEmpty(result) && result.Length > 1)
                    result = result.Substring(0, result.Length - 1);

                return result;
            }
        }
}

Give it a go and let me know how it works.